From db44bb36e1d85dfd9f8d8643e60b33831247184d Mon Sep 17 00:00:00 2001 From: junchen14 Date: Mon, 23 Oct 2023 09:14:40 +0300 Subject: [PATCH] remove wandb logs --- wandb/debug-cli.chenj0g.log | 0 wandb/debug-internal.log | 1 - wandb/debug.log | 1 - wandb/latest-run | 1 - .../files/conda-environment.yaml | 297 --- .../files/config.yaml | 38 - .../files/output.log | 177 -- .../files/requirements.txt | 246 --- .../files/wandb-metadata.json | 374 ---- .../files/wandb-summary.json | 1 - .../logs/debug-internal.log | 1843 ----------------- .../logs/debug.log | 27 - .../run-bxwpu63q.wandb | Bin 106822 -> 0 bytes .../files/conda-environment.yaml | 297 --- .../files/config.yaml | 38 - .../files/output.log | 164 -- .../files/requirements.txt | 246 --- .../files/wandb-metadata.json | 374 ---- .../files/wandb-summary.json | 1 - .../logs/debug-internal.log | 1701 --------------- .../logs/debug.log | 26 - .../run-141dzr9c.wandb | Bin 99174 -> 0 bytes .../files/conda-environment.yaml | 297 --- .../files/config.yaml | 38 - .../files/output.log | 2 - .../files/requirements.txt | 246 --- .../files/wandb-metadata.json | 374 ---- .../files/wandb-summary.json | 1 - .../logs/debug-internal.log | 1375 ------------ .../logs/debug.log | 26 - .../run-txn4i28h.wandb | Bin 48616 -> 0 bytes .../files/conda-environment.yaml | 297 --- .../files/config.yaml | 38 - .../files/output.log | 161 -- .../files/requirements.txt | 246 --- .../files/wandb-metadata.json | 374 ---- .../files/wandb-summary.json | 1 - .../logs/debug-internal.log | 1367 ------------ .../logs/debug.log | 26 - .../run-25yuqsbo.wandb | Bin 85770 -> 0 bytes 40 files changed, 10722 deletions(-) delete mode 100644 wandb/debug-cli.chenj0g.log delete mode 120000 wandb/debug-internal.log delete mode 120000 wandb/debug.log delete mode 120000 wandb/latest-run delete mode 100644 wandb/run-20231023_085438-bxwpu63q/files/conda-environment.yaml delete mode 100644 wandb/run-20231023_085438-bxwpu63q/files/config.yaml delete mode 100644 wandb/run-20231023_085438-bxwpu63q/files/output.log delete mode 100644 wandb/run-20231023_085438-bxwpu63q/files/requirements.txt delete mode 100644 wandb/run-20231023_085438-bxwpu63q/files/wandb-metadata.json delete mode 100644 wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json delete mode 100644 wandb/run-20231023_085438-bxwpu63q/logs/debug-internal.log delete mode 100644 wandb/run-20231023_085438-bxwpu63q/logs/debug.log delete mode 100644 wandb/run-20231023_085438-bxwpu63q/run-bxwpu63q.wandb delete mode 100644 wandb/run-20231023_090058-141dzr9c/files/conda-environment.yaml delete mode 100644 wandb/run-20231023_090058-141dzr9c/files/config.yaml delete mode 100644 wandb/run-20231023_090058-141dzr9c/files/output.log delete mode 100644 wandb/run-20231023_090058-141dzr9c/files/requirements.txt delete mode 100644 wandb/run-20231023_090058-141dzr9c/files/wandb-metadata.json delete mode 100644 wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json delete mode 100644 wandb/run-20231023_090058-141dzr9c/logs/debug-internal.log delete mode 100644 wandb/run-20231023_090058-141dzr9c/logs/debug.log delete mode 100644 wandb/run-20231023_090058-141dzr9c/run-141dzr9c.wandb delete mode 100644 wandb/run-20231023_091029-txn4i28h/files/conda-environment.yaml delete mode 100644 wandb/run-20231023_091029-txn4i28h/files/config.yaml delete mode 100644 wandb/run-20231023_091029-txn4i28h/files/output.log delete mode 100644 wandb/run-20231023_091029-txn4i28h/files/requirements.txt delete mode 100644 wandb/run-20231023_091029-txn4i28h/files/wandb-metadata.json delete mode 100644 wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json delete mode 100644 wandb/run-20231023_091029-txn4i28h/logs/debug-internal.log delete mode 100644 wandb/run-20231023_091029-txn4i28h/logs/debug.log delete mode 100644 wandb/run-20231023_091029-txn4i28h/run-txn4i28h.wandb delete mode 100644 wandb/run-20231023_091030-25yuqsbo/files/conda-environment.yaml delete mode 100644 wandb/run-20231023_091030-25yuqsbo/files/config.yaml delete mode 100644 wandb/run-20231023_091030-25yuqsbo/files/output.log delete mode 100644 wandb/run-20231023_091030-25yuqsbo/files/requirements.txt delete mode 100644 wandb/run-20231023_091030-25yuqsbo/files/wandb-metadata.json delete mode 100644 wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json delete mode 100644 wandb/run-20231023_091030-25yuqsbo/logs/debug-internal.log delete mode 100644 wandb/run-20231023_091030-25yuqsbo/logs/debug.log delete mode 100644 wandb/run-20231023_091030-25yuqsbo/run-25yuqsbo.wandb diff --git a/wandb/debug-cli.chenj0g.log b/wandb/debug-cli.chenj0g.log deleted file mode 100644 index e69de29..0000000 diff --git a/wandb/debug-internal.log b/wandb/debug-internal.log deleted file mode 120000 index 2b1e52e..0000000 --- a/wandb/debug-internal.log +++ /dev/null @@ -1 +0,0 @@ -run-20231023_091030-25yuqsbo/logs/debug-internal.log \ No newline at end of file diff --git a/wandb/debug.log b/wandb/debug.log deleted file mode 120000 index bce2fd2..0000000 --- a/wandb/debug.log +++ /dev/null @@ -1 +0,0 @@ -run-20231023_091030-25yuqsbo/logs/debug.log \ No newline at end of file diff --git a/wandb/latest-run b/wandb/latest-run deleted file mode 120000 index 4482a73..0000000 --- a/wandb/latest-run +++ /dev/null @@ -1 +0,0 @@ -run-20231023_091030-25yuqsbo \ No newline at end of file diff --git a/wandb/run-20231023_085438-bxwpu63q/files/conda-environment.yaml b/wandb/run-20231023_085438-bxwpu63q/files/conda-environment.yaml deleted file mode 100644 index cfced9e..0000000 --- a/wandb/run-20231023_085438-bxwpu63q/files/conda-environment.yaml +++ /dev/null @@ -1,297 +0,0 @@ -name: eye -channels: - - pytorch - - defaults -dependencies: - - _libgcc_mutex=0.1=main - - _openmp_mutex=5.1=1_gnu - - blas=1.0=mkl - - brotlipy=0.7.0=py39h27cfd23_1003 - - bzip2=1.0.8=h7b6447c_0 - - ca-certificates=2023.01.10=h06a4308_0 - - certifi=2022.12.7=py39h06a4308_0 - - cffi=1.15.1=py39h5eee18b_3 - - charset-normalizer=2.0.4=pyhd3eb1b0_0 - - cryptography=39.0.1=py39h9ce1e76_0 - - cudatoolkit=11.3.1=h2bc3f7f_2 - - ffmpeg=4.3=hf484d3e_0 - - flit-core=3.8.0=py39h06a4308_0 - - freetype=2.12.1=h4a9f257_0 - - giflib=5.2.1=h5eee18b_3 - - gmp=6.2.1=h295c915_3 - - gnutls=3.6.15=he1e5248_0 - - idna=3.4=py39h06a4308_0 - - intel-openmp=2021.4.0=h06a4308_3561 - - jpeg=9e=h5eee18b_1 - - lame=3.100=h7b6447c_0 - - lcms2=2.12=h3be6417_0 - - ld_impl_linux-64=2.38=h1181459_1 - - lerc=3.0=h295c915_0 - - libdeflate=1.17=h5eee18b_0 - - libffi=3.4.2=h6a678d5_6 - - libgcc-ng=11.2.0=h1234567_1 - - libgomp=11.2.0=h1234567_1 - - libiconv=1.16=h7f8727e_2 - - libidn2=2.3.2=h7f8727e_0 - - libpng=1.6.39=h5eee18b_0 - - libstdcxx-ng=11.2.0=h1234567_1 - - libtasn1=4.19.0=h5eee18b_0 - - libtiff=4.5.0=h6a678d5_2 - - libunistring=0.9.10=h27cfd23_0 - - libwebp=1.2.4=h11a3e52_1 - - libwebp-base=1.2.4=h5eee18b_1 - - lz4-c=1.9.4=h6a678d5_0 - - mkl=2021.4.0=h06a4308_640 - - mkl-service=2.4.0=py39h7f8727e_0 - - mkl_fft=1.3.1=py39hd3c417c_0 - - mkl_random=1.2.2=py39h51133e4_0 - - ncurses=6.4=h6a678d5_0 - - nettle=3.7.3=hbbd107a_1 - - numpy=1.23.5=py39h14f4228_0 - - numpy-base=1.23.5=py39h31eccc5_0 - - openh264=2.1.1=h4ff587b_0 - - openssl=1.1.1t=h7f8727e_0 - - pillow=9.4.0=py39h6a678d5_0 - - pip=23.0.1=py39h06a4308_0 - - pycparser=2.21=pyhd3eb1b0_0 - - pyopenssl=23.0.0=py39h06a4308_0 - - pysocks=1.7.1=py39h06a4308_0 - - python=3.9.16=h7a1cb2a_2 - - pytorch-mutex=1.0=cuda - - readline=8.2=h5eee18b_0 - - requests=2.28.1=py39h06a4308_1 - - setuptools=65.6.3=py39h06a4308_0 - - six=1.16.0=pyhd3eb1b0_1 - - sqlite=3.41.1=h5eee18b_0 - - tk=8.6.12=h1ccaba5_0 - - torchaudio=0.12.1=py39_cu113 - - torchvision=0.13.1=py39_cu113 - - typing_extensions=4.4.0=py39h06a4308_0 - - urllib3=1.26.15=py39h06a4308_0 - - wheel=0.38.4=py39h06a4308_0 - - xz=5.2.10=h5eee18b_1 - - zlib=1.2.13=h5eee18b_0 - - zstd=1.5.4=hc292b87_0 - - pip: - - accelerate==0.20.3 - - aiofiles==23.1.0 - - aiohttp==3.8.4 - - aiosignal==1.3.1 - - altair==4.2.2 - - antlr4-python3-runtime==4.9.3 - - anyio==3.6.2 - - appdirs==1.4.4 - - argon2-cffi==21.3.0 - - argon2-cffi-bindings==21.2.0 - - arrow==1.2.3 - - asttokens==2.2.1 - - async-timeout==4.0.2 - - attrs==22.2.0 - - backcall==0.2.0 - - beautifulsoup4==4.12.2 - - bitsandbytes==0.37.0 - - bleach==6.0.0 - - blis==0.7.9 - - braceexpand==0.1.7 - - catalogue==2.0.8 - - cchardet==2.1.7 - - chardet==5.1.0 - - click==8.1.3 - - cloudpickle==2.2.1 - - cmake==3.26.1 - - comm==0.1.3 - - confection==0.0.4 - - contourpy==1.0.7 - - cycler==0.11.0 - - cymem==2.0.7 - - datasets==2.13.1 - - debugpy==1.6.7 - - decorator==5.1.1 - - decord==0.6.0 - - defusedxml==0.7.1 - - dill==0.3.6 - - docker-pycreds==0.4.0 - - einops==0.6.0 - - entrypoints==0.4 - - executing==1.2.0 - - fairscale==0.4.13 - - fastapi==0.95.0 - - fastjsonschema==2.16.3 - - ffmpy==0.3.0 - - filelock==3.9.0 - - fire==0.5.0 - - fonttools==4.38.0 - - fqdn==1.5.1 - - frozenlist==1.3.3 - - fsspec==2023.4.0 - - gitdb==4.0.10 - - gitpython==3.1.31 - - gradio==3.24.1 - - gradio-client==0.0.8 - - h11==0.14.0 - - httpcore==0.16.3 - - httpx==0.23.3 - - huggingface-hub==0.16.4 - - imageio==2.31.1 - - importlib-metadata==6.2.0 - - importlib-resources==5.12.0 - - iopath==0.1.10 - - ipykernel==6.22.0 - - ipython==8.12.0 - - ipython-genutils==0.2.0 - - isoduration==20.11.0 - - jedi==0.18.2 - - jinja2==3.1.2 - - joblib==1.2.0 - - jsonpointer==2.3 - - jsonschema==4.17.3 - - jupyter-client==8.1.0 - - jupyter-core==5.3.0 - - jupyter-events==0.6.3 - - jupyter-server==2.5.0 - - jupyter-server-terminals==0.4.4 - - jupyterlab-pygments==0.2.2 - - kiwisolver==1.4.4 - - langcodes==3.3.0 - - lazy-loader==0.2 - - linkify-it-py==2.0.0 - - lit==16.0.0 - - llvmlite==0.39.1 - - loralib==0.1.1 - - markdown-it-py==2.2.0 - - markupsafe==2.1.2 - - matplotlib==3.7.0 - - matplotlib-inline==0.1.6 - - mdit-py-plugins==0.3.3 - - mdurl==0.1.2 - - mistune==2.0.5 - - mpmath==1.3.0 - - multidict==6.0.4 - - multiprocess==0.70.14 - - murmurhash==1.0.9 - - nbclassic==0.5.5 - - nbclient==0.7.3 - - nbconvert==7.3.0 - - nbformat==5.8.0 - - nest-asyncio==1.5.6 - - networkx==3.1 - - nltk==3.8.1 - - notebook==6.5.4 - - notebook-shim==0.2.2 - - numba==0.56.4 - - nvidia-cublas-cu11==11.10.3.66 - - nvidia-cuda-cupti-cu11==11.7.101 - - nvidia-cuda-nvrtc-cu11==11.7.99 - - nvidia-cuda-runtime-cu11==11.7.99 - - nvidia-cudnn-cu11==8.5.0.96 - - nvidia-cufft-cu11==10.9.0.58 - - nvidia-curand-cu11==10.2.10.91 - - nvidia-cusolver-cu11==11.4.0.1 - - nvidia-cusparse-cu11==11.7.4.91 - - nvidia-nccl-cu11==2.14.3 - - nvidia-nvtx-cu11==11.7.91 - - omegaconf==2.3.0 - - openai==0.27.0 - - opencv-python==4.7.0.72 - - orjson==3.8.9 - - packaging==23.0 - - pandas==2.0.0 - - pandocfilters==1.5.0 - - parso==0.8.3 - - pathlib==1.0.1 - - pathtools==0.1.2 - - pathy==0.10.1 - - peft==0.2.0 - - pexpect==4.8.0 - - pickleshare==0.7.5 - - platformdirs==3.2.0 - - policies==0.4.2 - - portalocker==2.7.0 - - preshed==3.0.8 - - progressbar2==4.2.0 - - prometheus-client==0.16.0 - - prompt-toolkit==3.0.38 - - protobuf==4.22.1 - - psutil==5.9.4 - - ptyprocess==0.7.0 - - pure-eval==0.2.2 - - pyarrow==12.0.1 - - pycocoevalcap==1.2 - - pycocotools==2.0.6 - - pycryptodome==3.17 - - pydantic==1.10.7 - - pydub==0.25.1 - - pygments==2.14.0 - - pynndescent==0.5.8 - - pyparsing==3.0.9 - - pyrsistent==0.19.3 - - python-dateutil==2.8.2 - - python-json-logger==2.0.7 - - python-multipart==0.0.6 - - python-utils==3.7.0 - - pytz==2023.3 - - pywavelets==1.4.1 - - pyyaml==6.0 - - pyzmq==25.0.2 - - regex==2022.10.31 - - rfc3339-validator==0.1.4 - - rfc3986==1.5.0 - - rfc3986-validator==0.1.1 - - safetensors==0.3.1 - - scikit-image==0.21.0 - - scikit-learn==1.2.2 - - scipy==1.10.1 - - seaborn==0.12.2 - - semantic-version==2.10.0 - - send2trash==1.8.0 - - sentence-transformers==2.2.2 - - sentencepiece==0.1.97 - - sentry-sdk==1.19.1 - - setproctitle==1.3.2 - - smart-open==6.3.0 - - smmap==5.0.0 - - sniffio==1.3.0 - - soupsieve==2.4 - - spacy==3.5.1 - - spacy-legacy==3.0.12 - - spacy-loggers==1.0.4 - - srsly==2.4.6 - - stack-data==0.6.2 - - starlette==0.26.1 - - submitit==1.4.5 - - sympy==1.11.1 - - tenacity==8.2.2 - - termcolor==2.3.0 - - terminado==0.17.1 - - thinc==8.1.9 - - threadpoolctl==3.1.0 - - tifffile==2023.4.12 - - timm==0.6.13 - - tinycss2==1.2.1 - - tokenizers==0.13.3 - - toolz==0.12.0 - - torch==2.0.0 - - tornado==6.2 - - tqdm==4.64.1 - - traitlets==5.9.0 - - transformers==4.31.0 - - triton==2.0.0 - - typer==0.7.0 - - tzdata==2023.3 - - uc-micro-py==1.0.1 - - umap-learn==0.5.3 - - uri-template==1.2.0 - - uvicorn==0.21.1 - - visual-genome==1.1.1 - - wandb==0.14.2 - - wasabi==1.1.1 - - wcwidth==0.2.6 - - webcolors==1.13 - - webdataset==0.2.48 - - webencodings==0.5.1 - - websocket-client==1.5.1 - - websockets==11.0.1 - - xxhash==3.2.0 - - yarl==1.8.2 - - zipp==3.14.0 -prefix: /home/chenj0g/anaconda3/envs/eye diff --git a/wandb/run-20231023_085438-bxwpu63q/files/config.yaml b/wandb/run-20231023_085438-bxwpu63q/files/config.yaml deleted file mode 100644 index 8f9f09d..0000000 --- a/wandb/run-20231023_085438-bxwpu63q/files/config.yaml +++ /dev/null @@ -1,38 +0,0 @@ -wandb_version: 1 - -_wandb: - desc: null - value: - python_version: 3.9.16 - cli_version: 0.14.2 - framework: huggingface - huggingface_version: 4.31.0 - is_jupyter_run: false - is_kaggle_kernel: false - start_time: 1698040478.151963 - t: - 1: - - 1 - - 11 - - 41 - - 49 - - 55 - - 63 - - 71 - 2: - - 1 - - 11 - - 41 - - 49 - - 55 - - 63 - - 71 - 3: - - 1 - - 13 - - 23 - 4: 3.9.16 - 5: 0.14.2 - 6: 4.31.0 - 8: - - 5 diff --git a/wandb/run-20231023_085438-bxwpu63q/files/output.log b/wandb/run-20231023_085438-bxwpu63q/files/output.log deleted file mode 100644 index 92a24d8..0000000 --- a/wandb/run-20231023_085438-bxwpu63q/files/output.log +++ /dev/null @@ -1,177 +0,0 @@ - -2023-10-23 08:54:52,781 [INFO] Start training -batch sizes [[2]] -module.llama_model.base_model.model.model.layers.0.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.0.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.0.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.0.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.1.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.1.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.1.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.1.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.2.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.2.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.2.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.2.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.3.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.3.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.3.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.3.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.4.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.4.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.4.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.4.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.5.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.5.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.5.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.5.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.6.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.6.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.6.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.6.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.7.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.7.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.7.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.7.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.8.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.8.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.8.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.8.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.9.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.9.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.9.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.9.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.10.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.10.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.10.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.10.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.11.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.11.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.11.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.11.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.12.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.12.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.12.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.12.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.13.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.13.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.13.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.13.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.14.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.14.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.14.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.14.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.15.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.15.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.15.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.15.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.16.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.16.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.16.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.16.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.17.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.17.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.17.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.17.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.18.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.18.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.18.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.18.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.19.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.19.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.19.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.19.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.20.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.20.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.20.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.20.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.21.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.21.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.21.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.21.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.22.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.22.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.22.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.22.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.23.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.23.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.23.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.23.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.24.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.24.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.24.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.24.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.25.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.25.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.25.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.25.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.26.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.26.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.26.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.26.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.27.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.27.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.27.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.27.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.28.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.28.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.28.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.28.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.29.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.29.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.29.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.29.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.30.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.30.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.30.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.30.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.31.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.31.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.31.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.31.self_attn.v_proj.lora_B.weight -module.llama_proj.weight -module.llama_proj.bias -2023-10-23 08:54:54,750 [INFO] dataset_ratios not specified, datasets will be concatenated (map-style datasets) or chained (webdataset.DataPipeline). -2023-10-23 08:54:54,750 [INFO] Loaded 12171 records for train split from the dataset. -2023-10-23 08:54:54,768 [INFO] number of trainable parameters: 56627200 -2023-10-23 08:54:54,769 [INFO] Start training epoch 0, 1000 iters per inner epoch. -Train: data epoch: [0] [ 0/1000] eta: 0:44:54 lr: 0.000001 loss: 1.3049 time: 2.6941 data: 0.0000 max mem: 33053 -/home/chenj0g/anaconda3/envs/eye/lib/python3.9/site-packages/torch/utils/checkpoint.py:31: UserWarning: None of the inputs have requires_grad=True. Gradients will be None - warnings.warn("None of the inputs have requires_grad=True. Gradients will be None") -Train: data epoch: [0] [ 50/1000] eta: 0:07:57 lr: 0.000001 loss: 1.2327 time: 0.4510 data: 0.0000 max mem: 38294 -Train: data epoch: [0] [ 100/1000] eta: 0:07:10 lr: 0.000002 loss: 1.1049 time: 0.4456 data: 0.0000 max mem: 39573 -Train: data epoch: [0] [ 150/1000] eta: 0:06:42 lr: 0.000002 loss: 1.2879 time: 0.4630 data: 0.0000 max mem: 40208 -Train: data epoch: [0] [ 200/1000] eta: 0:06:18 lr: 0.000003 loss: 1.0077 time: 0.4521 data: 0.0000 max mem: 40208 -Train: data epoch: [0] [ 250/1000] eta: 0:05:52 lr: 0.000003 loss: 1.4790 time: 0.4627 data: 0.0000 max mem: 41447 -Train: data epoch: [0] [ 300/1000] eta: 0:05:27 lr: 0.000004 loss: 1.2864 time: 0.4646 data: 0.0000 max mem: 41447 -Train: data epoch: [0] [ 350/1000] eta: 0:05:02 lr: 0.000004 loss: 1.1261 time: 0.4556 data: 0.0000 max mem: 41447 -Traceback (most recent call last): - File "/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/train.py", line 117, in - main() - File "/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/train.py", line 113, in main - runner.train() - File "/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/minigpt4/runners/runner_base.py", line 377, in train - train_stats = self.train_epoch(cur_epoch) - File "/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/minigpt4/runners/runner_base.py", line 437, in train_epoch - return self.task.train_epoch( - File "/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/minigpt4/tasks/base_task.py", line 116, in train_epoch - return self._train_inner_loop( - File "/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/minigpt4/tasks/base_task.py", line 221, in _train_inner_loop - loss = self.train_step(model=model, samples=samples) - File "/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/minigpt4/tasks/base_task.py", line 70, in train_step - loss = model(samples)["loss"] - File "/home/chenj0g/anaconda3/envs/eye/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl - return forward_call(*args, **kwargs) - File "/home/chenj0g/anaconda3/envs/eye/lib/python3.9/site-packages/torch/nn/parallel/distributed.py", line 1156, in forward - output = self._run_ddp_forward(*inputs, **kwargs) - File "/home/chenj0g/anaconda3/envs/eye/lib/python3.9/site-packages/torch/nn/parallel/distributed.py", line 1110, in _run_ddp_forward - return module_to_run(*inputs[0], **kwargs[0]) # type: ignore[index] - File "/home/chenj0g/anaconda3/envs/eye/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl - return forward_call(*args, **kwargs) - File "/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/minigpt4/models/minigpt_base.py", line 279, in forward - self.preparing_embedding(samples) - File "/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/minigpt4/models/minigpt_base.py", line 215, in preparing_embedding - img_embeds, img_atts = self.encode_img(samples["image"]) - File "/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/minigpt4/models/minigpt_v2.py", line 88, in encode_img - atts_llama = torch.ones(inputs_llama.size()[:-1], dtype=torch.long).to(image.device) -KeyboardInterrupt \ No newline at end of file diff --git a/wandb/run-20231023_085438-bxwpu63q/files/requirements.txt b/wandb/run-20231023_085438-bxwpu63q/files/requirements.txt deleted file mode 100644 index 75bc393..0000000 --- a/wandb/run-20231023_085438-bxwpu63q/files/requirements.txt +++ /dev/null @@ -1,246 +0,0 @@ -accelerate==0.20.3 -aiofiles==23.2.1 -aiohttp==3.8.4 -aiosignal==1.3.1 -altair==5.1.2 -antlr4-python3-runtime==4.9.3 -anyio==3.6.2 -appdirs==1.4.4 -argon2-cffi-bindings==21.2.0 -argon2-cffi==21.3.0 -arrow==1.2.3 -asttokens==2.2.1 -async-timeout==4.0.2 -attrs==22.2.0 -backcall==0.2.0 -beautifulsoup4==4.12.2 -bitsandbytes==0.37.0 -bleach==6.0.0 -blis==0.7.9 -braceexpand==0.1.7 -brotlipy==0.7.0 -catalogue==2.0.8 -cchardet==2.1.7 -certifi==2022.12.7 -cffi==1.15.1 -chardet==5.1.0 -charset-normalizer==2.0.4 -click==8.1.3 -cloudpickle==2.2.1 -cmake==3.26.1 -comm==0.1.3 -confection==0.0.4 -contourpy==1.0.7 -cryptography==39.0.1 -cycler==0.11.0 -cymem==2.0.7 -datasets==2.13.1 -debugpy==1.6.7 -decorator==5.1.1 -decord==0.6.0 -defusedxml==0.7.1 -dill==0.3.6 -docker-pycreds==0.4.0 -einops==0.6.0 -entrypoints==0.4 -executing==1.2.0 -fairscale==0.4.13 -fastapi==0.95.0 -fastjsonschema==2.16.3 -ffmpy==0.3.1 -filelock==3.9.0 -fire==0.5.0 -flit-core==3.8.0 -fonttools==4.38.0 -fqdn==1.5.1 -frozenlist==1.3.3 -fsspec==2023.9.2 -gitdb==4.0.10 -gitpython==3.1.31 -gradio-client==0.6.0 -gradio==3.24.1 -h11==0.14.0 -httpcore==0.18.0 -httpx==0.25.0 -huggingface-hub==0.18.0 -idna==3.4 -imageio==2.31.1 -importlib-metadata==6.2.0 -importlib-resources==5.12.0 -iopath==0.1.10 -ipykernel==6.22.0 -ipython-genutils==0.2.0 -ipython==8.12.0 -isoduration==20.11.0 -jedi==0.18.2 -jinja2==3.1.2 -joblib==1.2.0 -jsonpointer==2.3 -jsonschema==4.17.3 -jupyter-client==8.1.0 -jupyter-core==5.3.0 -jupyter-events==0.6.3 -jupyter-server-terminals==0.4.4 -jupyter-server==2.5.0 -jupyterlab-pygments==0.2.2 -kiwisolver==1.4.4 -langcodes==3.3.0 -lazy-loader==0.2 -linkify-it-py==2.0.2 -lit==16.0.0 -llvmlite==0.39.1 -loralib==0.1.1 -markdown-it-py==2.2.0 -markupsafe==2.1.2 -matplotlib-inline==0.1.6 -matplotlib==3.7.0 -mdit-py-plugins==0.3.3 -mdurl==0.1.2 -mistune==2.0.5 -mkl-fft==1.3.1 -mkl-random==1.2.2 -mkl-service==2.4.0 -mpmath==1.3.0 -multidict==6.0.4 -multiprocess==0.70.14 -murmurhash==1.0.9 -nbclassic==0.5.5 -nbclient==0.7.3 -nbconvert==7.3.0 -nbformat==5.8.0 -nest-asyncio==1.5.6 -networkx==3.1 -nltk==3.8.1 -notebook-shim==0.2.2 -notebook==6.5.4 -numba==0.56.4 -numpy==1.23.5 -nvidia-cublas-cu11==11.10.3.66 -nvidia-cuda-cupti-cu11==11.7.101 -nvidia-cuda-nvrtc-cu11==11.7.99 -nvidia-cuda-runtime-cu11==11.7.99 -nvidia-cudnn-cu11==8.5.0.96 -nvidia-cufft-cu11==10.9.0.58 -nvidia-curand-cu11==10.2.10.91 -nvidia-cusolver-cu11==11.4.0.1 -nvidia-cusparse-cu11==11.7.4.91 -nvidia-nccl-cu11==2.14.3 -nvidia-nvtx-cu11==11.7.91 -omegaconf==2.3.0 -openai==0.27.0 -opencv-python==4.7.0.72 -orjson==3.9.8 -packaging==23.0 -pandas==2.0.0 -pandocfilters==1.5.0 -parso==0.8.3 -pathlib==1.0.1 -pathtools==0.1.2 -pathy==0.10.1 -peft==0.2.0 -pexpect==4.8.0 -pickleshare==0.7.5 -pillow==9.4.0 -pip==23.0.1 -platformdirs==3.2.0 -policies==0.4.2 -portalocker==2.7.0 -preshed==3.0.8 -progressbar2==4.2.0 -prometheus-client==0.16.0 -prompt-toolkit==3.0.38 -protobuf==4.22.1 -psutil==5.9.4 -ptyprocess==0.7.0 -pure-eval==0.2.2 -pyarrow==12.0.1 -pycocoevalcap==1.2 -pycocotools==2.0.6 -pycparser==2.21 -pycryptodome==3.17 -pydantic==1.10.7 -pydub==0.25.1 -pygments==2.14.0 -pynndescent==0.5.8 -pyopenssl==23.0.0 -pyparsing==3.0.9 -pyrsistent==0.19.3 -pysocks==1.7.1 -python-dateutil==2.8.2 -python-json-logger==2.0.7 -python-multipart==0.0.6 -python-utils==3.7.0 -pytz==2023.3 -pywavelets==1.4.1 -pyyaml==6.0 -pyzmq==25.0.2 -regex==2022.10.31 -requests==2.28.1 -rfc3339-validator==0.1.4 -rfc3986-validator==0.1.1 -rfc3986==1.5.0 -rpe-index==1.2.0 -safetensors==0.3.1 -scikit-image==0.21.0 -scikit-learn==1.2.2 -scipy==1.10.1 -seaborn==0.12.2 -semantic-version==2.10.0 -send2trash==1.8.0 -sentence-transformers==2.2.2 -sentencepiece==0.1.97 -sentry-sdk==1.19.1 -setproctitle==1.3.2 -setuptools==65.6.3 -six==1.16.0 -smart-open==6.3.0 -smmap==5.0.0 -sniffio==1.3.0 -soupsieve==2.4 -spacy-legacy==3.0.12 -spacy-loggers==1.0.4 -spacy==3.5.1 -srsly==2.4.6 -stack-data==0.6.2 -starlette==0.26.1 -submitit==1.4.5 -sympy==1.11.1 -tenacity==8.2.2 -termcolor==2.3.0 -terminado==0.17.1 -thinc==8.1.9 -threadpoolctl==3.1.0 -tifffile==2023.4.12 -timm==0.6.13 -tinycss2==1.2.1 -tokenizers==0.13.3 -toolz==0.12.0 -torch==2.0.0 -torchaudio==0.12.1 -torchvision==0.13.1 -tornado==6.2 -tqdm==4.64.1 -traitlets==5.9.0 -transformers==4.31.0 -triton==2.0.0 -typer==0.7.0 -typing-extensions==4.4.0 -tzdata==2023.3 -uc-micro-py==1.0.2 -umap-learn==0.5.3 -uri-template==1.2.0 -urllib3==1.26.15 -uvicorn==0.21.1 -visual-genome==1.1.1 -wandb==0.14.2 -wasabi==1.1.1 -wcwidth==0.2.6 -webcolors==1.13 -webdataset==0.2.48 -webencodings==0.5.1 -websocket-client==1.5.1 -websockets==11.0.1 -wheel==0.38.4 -xxhash==3.2.0 -yarl==1.8.2 -zipp==3.14.0 \ No newline at end of file diff --git a/wandb/run-20231023_085438-bxwpu63q/files/wandb-metadata.json b/wandb/run-20231023_085438-bxwpu63q/files/wandb-metadata.json deleted file mode 100644 index 0b7590d..0000000 --- a/wandb/run-20231023_085438-bxwpu63q/files/wandb-metadata.json +++ /dev/null @@ -1,374 +0,0 @@ -{ - "os": "Linux-3.10.0-1160.76.1.el7.x86_64-x86_64-with-glibc2.17", - "python": "3.9.16", - "heartbeatAt": "2023-10-23T05:54:39.620251", - "startedAt": "2023-10-23T05:54:38.092408", - "docker": null, - "cuda": null, - "args": [ - "--job_name", - "minigpt4_v2_test", - "--cfg-path", - "train_configs/minigpt_v2_finetune.yaml" - ], - "state": "running", - "program": "/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/train.py", - "codePath": "train.py", - "git": { - "remote": "https://github.com/junchen14/MiniGPT-4_finetune", - "commit": "5c3ec8bb730b2e9df23b4bb3eb24108f9c04a8d6" - }, - "email": "jun.chen@kaust.edu.sa", - "root": "/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune", - "host": "gpu201-16-l", - "username": "chenj0g", - "executable": "/home/chenj0g/anaconda3/envs/eye/bin/python", - "cpu_count": 64, - "cpu_count_logical": 64, - "cpu_freq": { - "current": 1996.2770000000016, - "min": 0.0, - "max": 0.0 - }, - "cpu_freq_per_core": [ - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - } - ], - "disk": { - "total": 251.85773849487305, - "used": 7.9049072265625 - }, - "gpu": "NVIDIA A100-SXM4-80GB", - "gpu_count": 2, - "gpu_devices": [ - { - "name": "NVIDIA A100-SXM4-80GB", - "memory_total": 85899345920 - }, - { - "name": "NVIDIA A100-SXM4-80GB", - "memory_total": 85899345920 - } - ], - "memory": { - "total": 503.7154769897461 - } -} diff --git a/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json b/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json deleted file mode 100644 index 8c4dc99..0000000 --- a/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json +++ /dev/null @@ -1 +0,0 @@ -{"epoch": 0, "loss": 1.1899254322052002, "_timestamp": 1698040660.278502, "_runtime": 182.1265389919281, "_step": 355, "_wandb": {"runtime": 180}} \ No newline at end of file diff --git a/wandb/run-20231023_085438-bxwpu63q/logs/debug-internal.log b/wandb/run-20231023_085438-bxwpu63q/logs/debug-internal.log deleted file mode 100644 index ac6f9de..0000000 --- a/wandb/run-20231023_085438-bxwpu63q/logs/debug-internal.log +++ /dev/null @@ -1,1843 +0,0 @@ -2023-10-23 08:54:38,141 INFO StreamThr :49933 [internal.py:wandb_internal():86] W&B internal server running at pid: 49933, started at: 2023-10-23 08:54:38.140580 -2023-10-23 08:54:38,142 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status -2023-10-23 08:54:38,153 INFO WriterThread:49933 [datastore.py:open_for_write():85] open: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/run-bxwpu63q.wandb -2023-10-23 08:54:38,154 DEBUG SenderThread:49933 [sender.py:send():375] send: header -2023-10-23 08:54:38,561 DEBUG SenderThread:49933 [sender.py:send():375] send: run -2023-10-23 08:54:39,255 INFO SenderThread:49933 [dir_watcher.py:__init__():219] watching files in: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files -2023-10-23 08:54:39,255 INFO SenderThread:49933 [sender.py:_start_run_threads():1124] run started: bxwpu63q with start time 1698040478.151963 -2023-10-23 08:54:39,255 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:54:39,256 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:54:39,257 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: check_version -2023-10-23 08:54:39,257 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: check_version -2023-10-23 08:54:39,599 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: run_start -2023-10-23 08:54:39,604 DEBUG HandlerThread:49933 [system_info.py:__init__():31] System info init -2023-10-23 08:54:39,604 DEBUG HandlerThread:49933 [system_info.py:__init__():46] System info init done -2023-10-23 08:54:39,604 INFO HandlerThread:49933 [system_monitor.py:start():181] Starting system monitor -2023-10-23 08:54:39,604 INFO SystemMonitor:49933 [system_monitor.py:_start():145] Starting system asset monitoring threads -2023-10-23 08:54:39,604 INFO HandlerThread:49933 [system_monitor.py:probe():201] Collecting system info -2023-10-23 08:54:39,605 INFO SystemMonitor:49933 [interfaces.py:start():190] Started cpu monitoring -2023-10-23 08:54:39,605 INFO SystemMonitor:49933 [interfaces.py:start():190] Started disk monitoring -2023-10-23 08:54:39,605 INFO SystemMonitor:49933 [interfaces.py:start():190] Started gpu monitoring -2023-10-23 08:54:39,606 INFO SystemMonitor:49933 [interfaces.py:start():190] Started memory monitoring -2023-10-23 08:54:39,606 INFO SystemMonitor:49933 [interfaces.py:start():190] Started network monitoring -2023-10-23 08:54:39,620 DEBUG HandlerThread:49933 [system_info.py:probe():195] Probing system -2023-10-23 08:54:39,624 DEBUG HandlerThread:49933 [system_info.py:_probe_git():180] Probing git -2023-10-23 08:54:39,635 DEBUG HandlerThread:49933 [system_info.py:_probe_git():188] Probing git done -2023-10-23 08:54:39,635 DEBUG HandlerThread:49933 [system_info.py:probe():240] Probing system done -2023-10-23 08:54:39,635 DEBUG HandlerThread:49933 [system_monitor.py:probe():210] {'os': 'Linux-3.10.0-1160.76.1.el7.x86_64-x86_64-with-glibc2.17', 'python': '3.9.16', 'heartbeatAt': '2023-10-23T05:54:39.620251', 'startedAt': '2023-10-23T05:54:38.092408', 'docker': None, 'cuda': None, 'args': ('--job_name', 'minigpt4_v2_test', '--cfg-path', 'train_configs/minigpt_v2_finetune.yaml'), 'state': 'running', 'program': '/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/train.py', 'codePath': 'train.py', 'git': {'remote': 'https://github.com/junchen14/MiniGPT-4_finetune', 'commit': '5c3ec8bb730b2e9df23b4bb3eb24108f9c04a8d6'}, 'email': 'jun.chen@kaust.edu.sa', 'root': '/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune', 'host': 'gpu201-16-l', 'username': 'chenj0g', 'executable': '/home/chenj0g/anaconda3/envs/eye/bin/python', 'cpu_count': 64, 'cpu_count_logical': 64, 'cpu_freq': {'current': 1996.2770000000016, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}], 'disk': {'total': 251.85773849487305, 'used': 7.9049072265625}, 'gpu': 'NVIDIA A100-SXM4-80GB', 'gpu_count': 2, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}], 'memory': {'total': 503.7154769897461}} -2023-10-23 08:54:39,635 INFO HandlerThread:49933 [system_monitor.py:probe():211] Finished collecting system info -2023-10-23 08:54:39,635 INFO HandlerThread:49933 [system_monitor.py:probe():214] Publishing system info -2023-10-23 08:54:39,635 DEBUG HandlerThread:49933 [system_info.py:_save_pip():51] Saving list of pip packages installed into the current environment -2023-10-23 08:54:39,636 DEBUG HandlerThread:49933 [system_info.py:_save_pip():67] Saving pip packages done -2023-10-23 08:54:39,636 DEBUG HandlerThread:49933 [system_info.py:_save_conda():74] Saving list of conda packages installed into the current environment -2023-10-23 08:54:40,257 INFO Thread-12 :49933 [dir_watcher.py:_on_file_created():278] file/dir created: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/requirements.txt -2023-10-23 08:54:40,257 INFO Thread-12 :49933 [dir_watcher.py:_on_file_created():278] file/dir created: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/conda-environment.yaml -2023-10-23 08:54:40,257 INFO Thread-12 :49933 [dir_watcher.py:_on_file_created():278] file/dir created: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:54:52,732 DEBUG HandlerThread:49933 [system_info.py:_save_conda():86] Saving conda packages done -2023-10-23 08:54:52,734 INFO HandlerThread:49933 [system_monitor.py:probe():216] Finished publishing system info -2023-10-23 08:54:52,767 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:54:52,767 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: keepalive -2023-10-23 08:54:52,767 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:54:52,767 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: keepalive -2023-10-23 08:54:52,768 DEBUG SenderThread:49933 [sender.py:send():375] send: files -2023-10-23 08:54:52,768 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-metadata.json with policy now -2023-10-23 08:54:52,773 DEBUG SenderThread:49933 [sender.py:send():375] send: telemetry -2023-10-23 08:54:52,773 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 08:54:52,774 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: stop_status -2023-10-23 08:54:53,158 DEBUG SenderThread:49933 [sender.py:send():375] send: telemetry -2023-10-23 08:54:53,264 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/conda-environment.yaml -2023-10-23 08:54:53,265 INFO Thread-12 :49933 [dir_watcher.py:_on_file_created():278] file/dir created: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/output.log -2023-10-23 08:54:53,265 INFO Thread-12 :49933 [dir_watcher.py:_on_file_created():278] file/dir created: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-metadata.json -2023-10-23 08:54:53,539 INFO wandb-upload_0:49933 [upload_job.py:push():137] Uploaded file /tmp/tmpgw7o7efowandb/ck5f4n94-wandb-metadata.json -2023-10-23 08:54:54,159 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:54:55,284 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/output.log -2023-10-23 08:54:57,285 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/output.log -2023-10-23 08:54:57,463 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:54:57,464 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:54:57,465 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:54:57,466 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:54:57,900 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:54:57,901 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:54:57,901 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:54:57,902 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:54:58,286 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:54:58,411 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:54:58,412 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:54:58,412 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:54:58,413 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:54:58,806 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:54:58,806 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:54:58,806 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:54:58,807 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:54:59,237 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:54:59,237 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:54:59,238 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:54:59,238 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:54:59,238 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:54:59,287 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/output.log -2023-10-23 08:54:59,287 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:54:59,820 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:54:59,821 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:54:59,821 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:54:59,822 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:00,263 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:00,263 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:00,264 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:00,264 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:00,287 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:00,721 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:00,722 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:00,722 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:00,723 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:01,220 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:01,220 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:01,220 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:01,222 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:01,288 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:01,648 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:01,648 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:01,648 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:01,649 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:02,094 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:02,094 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:02,095 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:02,096 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:02,289 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:02,557 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:02,558 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:02,558 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:02,559 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:02,939 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:02,939 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:02,939 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:02,940 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:03,290 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:03,443 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:03,444 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:03,444 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:03,446 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:03,865 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:03,865 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:03,866 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:03,866 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:04,290 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:04,408 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:04,409 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:04,409 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:04,409 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:55:04,410 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:04,777 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:04,777 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:04,777 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:04,778 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:05,291 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:05,329 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:05,329 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:05,329 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:05,330 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:05,833 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:05,833 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:05,833 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:05,835 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:06,289 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:06,289 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:06,289 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:06,290 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:06,292 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:06,715 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:06,716 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:06,716 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:06,717 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:07,166 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:07,167 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:07,167 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:07,168 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:07,292 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:07,711 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:07,711 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:07,711 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:07,712 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:07,773 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 08:55:07,773 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: stop_status -2023-10-23 08:55:08,171 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:08,172 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:08,172 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:08,174 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:08,293 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:08,614 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:08,614 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:08,615 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:08,615 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:09,077 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:09,078 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:09,078 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:09,079 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:09,294 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:09,504 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:09,505 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:09,508 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:55:09,746 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:09,746 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:09,957 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:09,958 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:09,958 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:09,958 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:10,295 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/config.yaml -2023-10-23 08:55:10,295 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:10,419 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:10,420 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:10,420 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:10,422 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:10,856 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:10,857 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:10,857 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:10,858 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:11,295 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:11,409 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:11,410 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:11,410 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:11,411 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:11,858 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:11,858 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:11,858 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:11,859 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:12,296 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:12,399 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:12,400 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:12,400 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:12,401 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:12,782 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:12,783 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:12,783 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:12,785 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:13,206 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:13,207 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:13,207 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:13,212 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:13,297 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:13,591 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:13,592 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:13,592 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:13,593 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:14,044 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:14,045 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:14,045 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:14,046 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:14,297 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:14,498 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:14,498 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:14,499 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:14,499 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:14,976 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:14,976 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:14,977 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:14,978 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:55:14,979 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:15,298 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:15,451 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:15,451 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:15,451 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:15,452 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:15,888 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:15,888 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:15,888 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:15,889 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:16,273 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:16,274 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:16,274 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:16,275 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:16,299 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:16,756 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:16,757 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:16,757 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:16,758 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:17,201 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:17,201 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:17,201 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:17,203 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:17,299 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:17,636 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:17,636 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:17,637 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:17,638 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:18,145 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:18,146 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:18,146 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:18,147 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:18,300 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:18,546 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:18,547 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:18,547 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:18,548 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:18,985 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:18,986 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:18,986 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:18,987 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:19,305 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:19,490 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:19,491 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:19,491 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:19,493 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:19,933 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:19,933 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:19,934 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:19,934 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:20,305 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:20,428 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:20,429 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:20,429 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:20,430 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:55:20,430 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:20,875 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:20,876 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:20,876 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:20,877 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:21,302 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:21,302 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:21,302 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:21,303 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:21,306 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:21,736 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:21,737 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:21,737 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:21,739 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:22,180 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:22,181 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:22,181 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:22,182 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:22,307 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:22,668 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:22,668 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:22,668 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:22,669 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:22,776 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 08:55:22,776 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: stop_status -2023-10-23 08:55:23,106 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:23,107 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:23,107 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:23,108 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:23,307 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/output.log -2023-10-23 08:55:23,308 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:23,534 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:23,534 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:23,535 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:23,535 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:24,001 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:24,002 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:24,002 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:24,004 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:24,308 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:24,366 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:24,366 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:24,367 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:24,367 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:24,827 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:24,827 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:24,828 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:24,828 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:25,309 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:25,322 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:25,323 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:25,323 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:25,324 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:25,766 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:25,766 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:25,767 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:25,767 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:55:25,768 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:26,310 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:26,319 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:26,320 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:26,320 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:26,322 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:26,795 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:26,795 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:26,796 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:26,796 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:27,243 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:27,243 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:27,244 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:27,244 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:27,310 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:27,698 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:27,699 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:27,699 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:27,700 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:28,121 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:28,122 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:28,122 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:28,123 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:28,311 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:28,564 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:28,565 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:28,565 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:28,566 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:29,131 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:29,131 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:29,131 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:29,132 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:29,312 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:29,575 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:29,576 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:29,576 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:29,577 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:30,038 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:30,038 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:30,038 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:30,039 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:30,312 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:30,557 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:30,557 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:30,558 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:30,558 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:30,980 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:30,981 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:30,981 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:30,982 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:55:30,983 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:31,313 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:31,485 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:31,486 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:31,486 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:31,487 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:31,960 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:31,960 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:31,960 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:31,961 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:32,314 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:32,430 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:32,431 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:32,431 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:32,432 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:32,860 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:32,861 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:32,861 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:32,862 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:33,314 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:33,320 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:33,320 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:33,320 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:33,322 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:33,706 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:33,706 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:33,706 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:33,707 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:34,180 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:34,181 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:34,181 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:34,182 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:34,315 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:34,649 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:34,650 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:34,650 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:34,651 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:35,152 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:35,152 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:35,152 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:35,153 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:35,316 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:35,574 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:35,575 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:35,575 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:35,577 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:35,944 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:35,945 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:35,945 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:35,946 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:36,317 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:36,373 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:36,373 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:36,374 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:36,374 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:55:36,375 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:36,842 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:36,842 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:36,842 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:36,843 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:37,289 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:37,290 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:37,290 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:37,291 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:37,317 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:37,797 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 08:55:37,798 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: stop_status -2023-10-23 08:55:37,801 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:38,094 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:38,095 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:38,096 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:38,167 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:38,168 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:38,168 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:38,169 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:38,318 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:38,562 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:38,563 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:38,563 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:38,564 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:39,040 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:39,041 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:39,041 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:39,042 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:39,318 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:39,479 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:39,480 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:39,480 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:39,481 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:39,606 DEBUG SystemMonitor:49933 [system_monitor.py:_start():159] Starting system metrics aggregation loop -2023-10-23 08:55:39,608 DEBUG SenderThread:49933 [sender.py:send():375] send: stats -2023-10-23 08:55:39,951 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:39,951 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:39,952 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:39,952 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:40,319 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:40,383 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:40,384 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:40,384 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:40,385 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:40,806 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:40,807 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:40,807 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:40,808 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:41,203 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:41,204 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:41,204 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:41,205 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:41,320 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:41,639 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:41,640 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:41,640 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:41,640 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:55:41,641 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:42,253 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:42,254 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:42,254 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:42,255 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:42,321 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:42,693 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:42,694 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:42,694 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:42,695 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:43,092 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:43,093 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:43,093 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:43,094 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:43,321 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:43,636 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:43,637 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:43,637 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:43,638 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:44,130 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:44,130 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:44,130 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:44,131 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:44,322 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:44,514 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:44,515 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:44,515 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:44,516 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:44,972 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:44,972 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:44,972 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:44,973 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:45,323 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/output.log -2023-10-23 08:55:45,323 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:45,448 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:45,449 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:45,449 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:45,450 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:46,031 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:46,032 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:46,032 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:46,033 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:46,323 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:46,601 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:46,602 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:46,602 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:46,603 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:47,062 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:47,062 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:47,063 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:47,063 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:55:47,064 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:47,324 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:47,476 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:47,477 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:47,477 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:47,478 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:47,941 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:47,942 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:47,942 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:47,943 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:48,325 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:48,418 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:48,419 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:48,419 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:48,421 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:48,832 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:48,833 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:48,833 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:48,834 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:49,298 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:49,298 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:49,298 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:49,299 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:49,325 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:49,693 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:49,693 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:49,694 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:49,694 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:50,169 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:50,169 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:50,170 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:50,170 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:50,326 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:50,615 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:50,616 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:50,616 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:50,617 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:51,077 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:51,077 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:51,078 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:51,078 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:51,327 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:51,523 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:51,523 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:51,524 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:51,524 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:51,964 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:51,964 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:51,964 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:51,965 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:52,328 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:52,448 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:52,448 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:52,448 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:52,448 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:55:52,449 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:52,824 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 08:55:52,825 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: stop_status -2023-10-23 08:55:52,828 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:53,090 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:53,091 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:53,091 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:53,303 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:53,303 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:53,303 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:53,304 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:53,329 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:53,780 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:53,781 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:53,781 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:53,782 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:54,253 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:54,253 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:54,253 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:54,254 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:54,329 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:54,715 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:54,716 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:54,716 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:54,716 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:55,267 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:55,268 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:55,268 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:55,269 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:55,330 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:55,712 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:55,712 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:55,713 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:55,713 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:56,123 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:56,124 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:56,124 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:56,125 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:56,331 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:56,598 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:56,599 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:56,599 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:56,600 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:57,017 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:57,017 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:57,018 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:57,020 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:57,331 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:57,522 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:57,523 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:57,523 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:57,523 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:55:57,524 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:57,966 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:57,967 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:57,967 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:57,968 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:58,332 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:58,394 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:58,395 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:58,395 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:58,396 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:58,841 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:58,842 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:58,842 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:58,844 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:59,237 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:59,237 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:59,237 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:59,238 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:59,333 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:59,661 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:59,662 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:59,662 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:59,663 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:00,169 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:00,170 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:00,170 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:00,171 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:00,333 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:00,598 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:00,598 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:00,598 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:00,599 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:01,037 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:01,037 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:01,038 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:01,038 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:01,334 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:01,505 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:01,506 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:01,506 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:01,507 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:01,964 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:01,965 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:01,965 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:01,966 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:02,335 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:02,458 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:02,458 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:02,459 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:02,459 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:03,003 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:03,004 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:03,004 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:03,004 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:56:03,005 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:03,335 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:03,509 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:03,509 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:03,509 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:03,510 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:04,013 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:04,014 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:04,014 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:04,015 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:04,336 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:04,501 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:04,501 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:04,502 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:04,502 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:05,007 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:05,008 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:05,008 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:05,009 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:05,337 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:05,368 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:05,369 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:05,369 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:05,370 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:05,842 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:05,842 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:05,843 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:05,843 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:06,278 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:06,278 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:06,279 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:06,279 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:06,338 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:06,726 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:06,726 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:06,726 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:06,727 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:07,244 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:07,244 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:07,244 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:07,245 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:07,338 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/output.log -2023-10-23 08:56:07,338 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:07,801 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:07,802 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:07,802 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:07,803 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:07,824 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 08:56:07,824 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: stop_status -2023-10-23 08:56:08,067 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:56:08,238 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:08,239 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:08,239 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:08,240 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:08,339 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:08,634 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:08,634 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:08,634 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:08,635 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:09,178 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:09,179 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:09,179 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:09,181 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:09,340 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:09,609 DEBUG SenderThread:49933 [sender.py:send():375] send: stats -2023-10-23 08:56:09,736 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:09,736 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:09,737 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:09,737 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:10,341 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:10,347 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:10,348 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:10,348 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:10,349 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:10,802 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:10,802 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:10,802 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:10,803 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:11,304 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:11,304 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:11,304 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:11,305 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:11,341 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:11,848 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:11,849 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:11,849 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:11,850 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:12,233 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:12,234 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:12,234 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:12,235 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:12,342 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:12,700 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:12,701 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:12,701 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:12,702 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:13,160 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:13,161 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:13,161 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:13,161 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:56:13,162 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:13,343 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:13,679 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:13,679 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:13,680 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:13,680 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:14,231 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:14,232 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:14,232 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:14,233 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:14,343 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:14,633 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:14,633 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:14,633 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:14,634 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:15,076 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:15,077 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:15,077 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:15,078 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:15,344 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:15,537 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:15,537 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:15,537 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:15,538 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:16,057 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:16,057 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:16,057 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:16,058 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:16,345 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:16,553 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:16,553 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:16,553 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:16,554 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:16,938 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:16,939 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:16,939 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:16,939 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:17,346 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:17,396 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:17,397 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:17,397 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:17,398 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:17,875 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:17,875 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:17,875 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:17,876 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:18,346 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:18,351 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:18,351 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:18,352 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:18,352 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:56:18,352 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:18,834 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:18,834 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:18,834 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:18,835 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:19,296 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:19,297 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:19,297 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:19,299 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:19,347 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:19,760 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:19,761 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:19,761 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:19,762 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:20,210 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:20,210 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:20,211 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:20,211 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:20,348 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:20,711 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:20,711 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:20,711 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:20,712 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:21,219 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:21,219 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:21,219 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:21,220 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:21,348 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:21,666 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:21,666 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:21,666 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:21,667 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:22,136 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:22,136 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:22,136 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:22,137 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:22,349 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:22,575 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:22,575 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:22,575 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:22,576 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:22,824 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 08:56:22,824 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: stop_status -2023-10-23 08:56:22,970 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:23,033 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:23,033 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:23,034 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:23,350 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:23,400 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:23,400 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:23,401 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:23,401 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:56:23,401 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:23,858 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:23,859 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:23,859 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:23,859 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:24,235 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:24,235 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:24,235 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:24,236 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:24,350 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:24,606 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:24,606 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:24,606 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:24,608 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:25,070 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:25,070 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:25,071 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:25,071 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:25,351 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:25,408 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:25,409 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:25,409 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:25,409 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:25,982 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:25,983 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:25,983 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:25,984 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:26,352 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:26,477 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:26,477 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:26,477 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:26,478 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:26,855 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:26,856 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:26,856 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:26,857 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:27,320 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:27,320 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:27,320 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:27,321 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:27,353 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:27,813 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:27,814 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:27,814 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:27,815 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:28,250 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:28,251 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:28,251 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:28,252 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:28,353 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:28,793 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:28,794 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:28,794 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:28,794 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:56:28,795 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:29,239 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:29,239 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:29,239 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:29,240 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:29,354 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:29,753 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:29,753 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:29,754 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:29,756 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:30,214 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:30,214 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:30,214 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:30,215 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:30,355 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:30,645 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:30,646 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:30,646 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:30,647 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:31,079 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:31,079 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:31,079 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:31,080 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:31,355 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/output.log -2023-10-23 08:56:31,356 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:31,559 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:31,559 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:31,560 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:31,560 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:32,005 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:32,006 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:32,006 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:32,007 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:32,356 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:32,416 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:32,417 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:32,417 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:32,418 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:32,887 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:32,887 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:32,887 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:32,888 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:33,357 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:33,364 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:33,364 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:33,365 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:33,365 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:33,877 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:33,877 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:33,878 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:33,878 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:56:33,878 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:34,322 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:34,323 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:34,323 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:34,324 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:34,357 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:34,791 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:34,791 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:34,791 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:34,792 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:35,253 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:35,253 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:35,253 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:35,254 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:35,358 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:35,829 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:35,829 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:35,829 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:35,830 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:36,288 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:36,288 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:36,288 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:36,289 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:36,359 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:36,651 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:36,651 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:36,652 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:36,652 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:37,109 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:37,110 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:37,110 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:37,110 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:37,360 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:37,552 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:37,553 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:37,553 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:37,554 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:37,824 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 08:56:37,824 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: stop_status -2023-10-23 08:56:37,938 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:38,058 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:38,058 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:38,059 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:38,360 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:38,384 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:38,385 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:38,385 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:38,386 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:38,766 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:38,766 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:38,767 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:38,767 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:39,310 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:39,311 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:39,311 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:39,311 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:56:39,312 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:39,361 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:39,610 DEBUG SenderThread:49933 [sender.py:send():375] send: stats -2023-10-23 08:56:39,694 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:39,694 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:39,694 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:39,695 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:40,315 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:40,316 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:40,316 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:40,318 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:40,362 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:40,700 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:40,700 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:40,700 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:40,701 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:41,072 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:41,073 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:41,073 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:41,074 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:41,363 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:41,441 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:41,441 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:41,441 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:41,442 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:41,997 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:41,998 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:41,998 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:41,999 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:42,363 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:42,518 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:42,518 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:42,519 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:42,519 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:42,945 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:42,945 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:42,945 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:42,946 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:43,364 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:43,370 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:43,371 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:43,371 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:43,372 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:43,949 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:43,950 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:43,950 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:43,951 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:44,365 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:44,417 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:44,417 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:44,418 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:44,418 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:56:44,418 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:44,871 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:44,872 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:44,872 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:44,873 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:45,266 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:45,266 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:45,266 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:45,267 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:45,365 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:45,713 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:45,714 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:45,714 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:45,715 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:46,189 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:46,190 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:46,190 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:46,191 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:46,366 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:46,813 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:46,814 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:46,814 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:46,815 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:47,246 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:47,246 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:47,246 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:47,247 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:47,367 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:47,691 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:47,692 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:47,692 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:47,692 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:48,168 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:48,168 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:48,169 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:48,169 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:48,368 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:48,636 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:48,636 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:48,636 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:48,637 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:49,007 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:49,008 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:49,008 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:49,009 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:49,368 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:49,559 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:49,560 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:49,560 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:49,560 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:56:49,561 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:49,999 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:50,000 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:50,000 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:50,001 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:50,369 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:50,444 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:50,445 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:50,445 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:50,447 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:50,891 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:50,892 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:50,892 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:50,892 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:51,356 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:51,356 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:51,357 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:51,357 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:51,370 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:51,800 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:51,800 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:51,800 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:51,801 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:52,222 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:52,223 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:52,223 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:52,224 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:52,370 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:52,625 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:52,626 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:52,626 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:52,626 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:52,824 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 08:56:52,824 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: stop_status -2023-10-23 08:56:53,127 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:53,128 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:53,128 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:53,129 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:53,371 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:53,564 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:53,564 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:53,565 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:53,565 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:54,010 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:54,010 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:54,010 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:54,011 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:54,372 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:54,462 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:54,462 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:54,462 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:54,463 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:54,922 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:54,922 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:54,922 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:54,923 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:56:54,923 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:55,372 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/output.log -2023-10-23 08:56:55,373 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:55,397 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:55,398 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:55,398 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:55,399 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:55,793 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:55,793 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:55,793 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:55,794 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:56,221 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:56,222 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:56,222 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:56,223 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:56,373 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:56,724 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:56,724 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:56,724 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:56,725 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:57,198 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:57,199 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:57,199 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:57,200 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:57,374 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:57,582 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:57,583 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:57,583 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:57,584 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:58,104 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:58,105 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:58,105 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:58,106 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:58,375 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:58,583 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:58,583 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:58,584 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:58,584 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:59,050 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:59,050 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:59,050 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:59,051 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:59,375 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:59,506 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:59,506 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:59,506 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:59,507 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:59,899 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:59,900 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:59,900 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:59,901 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:00,339 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:00,339 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:00,339 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:00,339 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:57:00,340 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:00,376 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:00,815 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:00,815 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:00,815 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:00,817 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:01,244 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:01,244 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:01,244 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:01,245 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:01,377 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:01,685 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:01,685 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:01,685 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:01,686 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:02,058 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:02,058 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:02,058 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:02,059 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:02,378 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:02,532 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:02,533 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:02,533 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:02,534 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:03,067 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:03,068 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:03,068 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:03,069 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:03,378 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:03,445 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:03,446 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:03,446 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:03,446 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:03,865 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:03,865 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:03,866 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:03,866 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:04,264 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:04,265 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:04,265 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:04,266 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:04,379 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:04,770 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:04,771 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:04,771 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:04,772 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:05,295 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:05,296 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:05,296 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:05,297 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:05,380 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:05,740 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:05,740 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:05,740 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:05,741 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:57:05,741 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:06,189 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:06,189 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:06,189 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:06,190 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:06,381 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:06,625 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:06,625 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:06,625 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:06,626 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:07,015 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:07,015 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:07,015 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:07,016 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:07,381 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:07,443 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:07,443 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:07,444 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:07,444 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:07,824 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 08:57:07,824 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: stop_status -2023-10-23 08:57:07,988 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:08,025 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:08,025 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:08,026 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:08,382 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:08,503 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:08,503 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:08,503 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:08,504 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:09,023 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:09,024 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:09,024 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:09,025 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:09,383 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:09,581 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:09,581 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:09,581 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:09,582 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:09,611 DEBUG SenderThread:49933 [sender.py:send():375] send: stats -2023-10-23 08:57:10,096 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:10,097 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:10,097 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:10,098 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:10,383 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:10,544 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:10,545 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:10,545 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:10,545 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:10,991 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:10,992 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:10,992 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:10,993 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:57:10,994 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:11,384 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:11,446 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:11,447 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:11,447 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:11,448 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:11,822 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:11,822 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:11,822 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:11,823 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:12,296 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:12,296 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:12,297 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:12,298 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:12,385 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:12,699 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:12,700 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:12,700 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:12,701 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:13,126 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:13,126 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:13,126 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:13,127 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:13,386 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:13,717 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:13,717 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:13,717 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:13,718 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:14,196 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:14,197 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:14,197 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:14,197 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:14,386 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:14,642 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:14,642 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:14,642 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:14,643 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:15,075 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:15,076 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:15,076 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:15,077 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:15,387 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:15,480 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:15,480 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:15,481 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:15,481 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:15,958 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:15,959 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:15,959 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:15,960 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:16,387 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:16,387 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:16,388 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:16,388 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:57:16,388 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:16,389 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:16,864 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:16,864 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:16,864 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:16,865 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:17,305 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:17,306 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:17,306 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:17,306 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:17,389 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/output.log -2023-10-23 08:57:17,389 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:17,693 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:17,693 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:17,693 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:17,694 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:18,116 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:18,117 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:18,117 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:18,120 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:18,390 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:18,539 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:18,540 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:18,540 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:18,541 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:18,934 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:18,934 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:18,934 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:18,935 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:19,323 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:19,323 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:19,323 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:19,324 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:19,390 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:19,750 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:19,751 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:19,751 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:19,752 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:20,332 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:20,333 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:20,333 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:20,334 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:20,391 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:20,809 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:20,810 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:20,810 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:20,811 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:21,260 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:21,261 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:21,261 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:21,263 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:21,392 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:21,696 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:21,697 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:21,697 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:21,697 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:57:21,698 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:22,144 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:22,144 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:22,144 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:22,145 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:22,393 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:22,603 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:22,603 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:22,604 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:22,604 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:22,824 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 08:57:22,824 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: stop_status -2023-10-23 08:57:23,048 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:23,049 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:23,049 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:23,049 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:23,393 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:23,490 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:23,491 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:23,491 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:23,492 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:23,913 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:23,913 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:23,913 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:23,914 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:24,394 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:24,442 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:24,442 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:24,442 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:24,443 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:24,880 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:24,881 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:24,881 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:24,882 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:25,325 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:25,325 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:25,325 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:25,326 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:25,395 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:25,722 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:25,722 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:25,722 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:25,723 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:26,151 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:26,152 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:26,152 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:26,153 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:26,395 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:26,674 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:26,675 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:26,675 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:26,676 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:27,104 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:27,105 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:27,105 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:27,105 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:57:27,106 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:27,396 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:27,631 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:27,632 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:27,632 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:27,633 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:28,074 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:28,075 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:28,075 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:28,076 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:28,397 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:28,497 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:28,498 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:28,498 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:28,499 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:28,949 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:28,949 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:28,950 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:28,950 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:29,397 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:29,419 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:29,419 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:29,420 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:29,420 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:29,980 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:29,980 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:29,981 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:29,981 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:30,380 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:30,380 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:30,381 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:30,381 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:30,398 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:30,968 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:30,969 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:30,969 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:30,970 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:31,344 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:31,345 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:31,345 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:31,347 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:31,399 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:31,774 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:31,775 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:31,775 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:31,776 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:32,222 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:32,222 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:32,223 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:32,223 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:57:32,224 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:32,399 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:32,663 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:32,663 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:32,664 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:32,664 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:33,145 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:33,146 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:33,146 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:33,147 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:33,400 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:33,590 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:33,591 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:33,591 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:33,591 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:34,072 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:34,072 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:34,073 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:34,073 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:34,401 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:34,556 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:34,556 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:34,556 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:34,557 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:34,972 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:34,973 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:34,973 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:34,974 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:35,401 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:35,418 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:35,419 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:35,419 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:35,420 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:35,866 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:35,867 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:35,867 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:35,868 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:36,335 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:36,335 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:36,335 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:36,336 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:36,402 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:36,753 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:36,753 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:36,754 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:36,754 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:37,219 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:37,219 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:37,219 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:37,220 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:37,403 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:37,683 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:37,684 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:37,684 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:37,684 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:57:37,685 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:37,824 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 08:57:37,824 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: stop_status -2023-10-23 08:57:38,062 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:38,062 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:38,062 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:38,063 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:38,403 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:38,485 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:38,485 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:38,486 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:38,486 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:38,929 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:38,930 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:38,930 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:38,931 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:39,375 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:39,375 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:39,376 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:39,377 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:39,404 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/output.log -2023-10-23 08:57:39,404 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:39,612 DEBUG SenderThread:49933 [sender.py:send():375] send: stats -2023-10-23 08:57:39,816 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:39,816 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:39,816 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:39,817 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:40,280 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:40,281 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:40,281 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:40,282 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:40,359 DEBUG SenderThread:49933 [sender.py:send():375] send: exit -2023-10-23 08:57:40,359 INFO SenderThread:49933 [sender.py:send_exit():598] handling exit code: 255 -2023-10-23 08:57:40,360 INFO SenderThread:49933 [sender.py:send_exit():600] handling runtime: 180 -2023-10-23 08:57:40,360 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:40,360 INFO SenderThread:49933 [sender.py:send_exit():606] send defer -2023-10-23 08:57:40,361 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: defer -2023-10-23 08:57:40,361 INFO HandlerThread:49933 [handler.py:handle_request_defer():170] handle defer: 0 -2023-10-23 08:57:40,361 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: defer -2023-10-23 08:57:40,361 INFO SenderThread:49933 [sender.py:send_request_defer():622] handle sender defer: 0 -2023-10-23 08:57:40,361 INFO SenderThread:49933 [sender.py:transition_state():626] send defer: 1 -2023-10-23 08:57:40,361 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: defer -2023-10-23 08:57:40,361 INFO HandlerThread:49933 [handler.py:handle_request_defer():170] handle defer: 1 -2023-10-23 08:57:40,361 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: defer -2023-10-23 08:57:40,361 INFO SenderThread:49933 [sender.py:send_request_defer():622] handle sender defer: 1 -2023-10-23 08:57:40,361 INFO SenderThread:49933 [sender.py:transition_state():626] send defer: 2 -2023-10-23 08:57:40,361 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: defer -2023-10-23 08:57:40,361 INFO HandlerThread:49933 [handler.py:handle_request_defer():170] handle defer: 2 -2023-10-23 08:57:40,361 INFO HandlerThread:49933 [system_monitor.py:finish():190] Stopping system monitor -2023-10-23 08:57:40,362 DEBUG SystemMonitor:49933 [system_monitor.py:_start():166] Finished system metrics aggregation loop -2023-10-23 08:57:40,362 INFO HandlerThread:49933 [interfaces.py:finish():202] Joined cpu monitor -2023-10-23 08:57:40,362 DEBUG SystemMonitor:49933 [system_monitor.py:_start():170] Publishing last batch of metrics -2023-10-23 08:57:40,362 INFO HandlerThread:49933 [interfaces.py:finish():202] Joined disk monitor -2023-10-23 08:57:40,408 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:40,474 INFO HandlerThread:49933 [interfaces.py:finish():202] Joined gpu monitor -2023-10-23 08:57:40,474 INFO HandlerThread:49933 [interfaces.py:finish():202] Joined memory monitor -2023-10-23 08:57:40,474 INFO HandlerThread:49933 [interfaces.py:finish():202] Joined network monitor -2023-10-23 08:57:40,475 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: defer -2023-10-23 08:57:40,475 INFO SenderThread:49933 [sender.py:send_request_defer():622] handle sender defer: 2 -2023-10-23 08:57:40,475 INFO SenderThread:49933 [sender.py:transition_state():626] send defer: 3 -2023-10-23 08:57:40,475 DEBUG SenderThread:49933 [sender.py:send():375] send: stats -2023-10-23 08:57:40,475 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: defer -2023-10-23 08:57:40,475 INFO HandlerThread:49933 [handler.py:handle_request_defer():170] handle defer: 3 -2023-10-23 08:57:40,475 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: defer -2023-10-23 08:57:40,475 INFO SenderThread:49933 [sender.py:send_request_defer():622] handle sender defer: 3 -2023-10-23 08:57:40,476 INFO SenderThread:49933 [sender.py:transition_state():626] send defer: 4 -2023-10-23 08:57:40,476 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: defer -2023-10-23 08:57:40,476 INFO HandlerThread:49933 [handler.py:handle_request_defer():170] handle defer: 4 -2023-10-23 08:57:40,476 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: defer -2023-10-23 08:57:40,476 INFO SenderThread:49933 [sender.py:send_request_defer():622] handle sender defer: 4 -2023-10-23 08:57:40,476 INFO SenderThread:49933 [sender.py:transition_state():626] send defer: 5 -2023-10-23 08:57:40,476 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: defer -2023-10-23 08:57:40,476 INFO HandlerThread:49933 [handler.py:handle_request_defer():170] handle defer: 5 -2023-10-23 08:57:40,476 DEBUG SenderThread:49933 [sender.py:send():375] send: summary -2023-10-23 08:57:40,477 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:40,477 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: defer -2023-10-23 08:57:40,477 INFO SenderThread:49933 [sender.py:send_request_defer():622] handle sender defer: 5 -2023-10-23 08:57:40,477 INFO SenderThread:49933 [sender.py:transition_state():626] send defer: 6 -2023-10-23 08:57:40,477 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: defer -2023-10-23 08:57:40,477 INFO HandlerThread:49933 [handler.py:handle_request_defer():170] handle defer: 6 -2023-10-23 08:57:40,477 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: defer -2023-10-23 08:57:40,477 INFO SenderThread:49933 [sender.py:send_request_defer():622] handle sender defer: 6 -2023-10-23 08:57:40,477 INFO SenderThread:49933 [sender.py:transition_state():626] send defer: 7 -2023-10-23 08:57:40,478 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:57:40,478 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: defer -2023-10-23 08:57:40,478 INFO HandlerThread:49933 [handler.py:handle_request_defer():170] handle defer: 7 -2023-10-23 08:57:40,478 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: defer -2023-10-23 08:57:40,478 INFO SenderThread:49933 [sender.py:send_request_defer():622] handle sender defer: 7 -2023-10-23 08:57:41,360 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: poll_exit -2023-10-23 08:57:41,409 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/output.log -2023-10-23 08:57:41,409 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:42,956 INFO SenderThread:49933 [sender.py:transition_state():626] send defer: 8 -2023-10-23 08:57:42,956 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: poll_exit -2023-10-23 08:57:42,956 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: defer -2023-10-23 08:57:42,956 INFO HandlerThread:49933 [handler.py:handle_request_defer():170] handle defer: 8 -2023-10-23 08:57:42,956 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: defer -2023-10-23 08:57:42,956 INFO SenderThread:49933 [sender.py:send_request_defer():622] handle sender defer: 8 -2023-10-23 08:57:43,015 INFO SenderThread:49933 [sender.py:transition_state():626] send defer: 9 -2023-10-23 08:57:43,015 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: defer -2023-10-23 08:57:43,015 INFO HandlerThread:49933 [handler.py:handle_request_defer():170] handle defer: 9 -2023-10-23 08:57:43,015 DEBUG SenderThread:49933 [sender.py:send():375] send: artifact -2023-10-23 08:57:43,410 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/output.log -2023-10-23 08:57:44,837 INFO wandb-upload_1:49933 [upload_job.py:push():95] Uploaded file /home/chenj0g/.local/share/wandb/artifacts/staging/tmp3h8gxbk8 -2023-10-23 08:57:44,842 INFO wandb-upload_0:49933 [upload_job.py:push():95] Uploaded file /home/chenj0g/.local/share/wandb/artifacts/staging/tmpv6vdw94w -2023-10-23 08:57:46,226 INFO SenderThread:49933 [sender.py:send_artifact():1474] sent artifact job-https___github.com_junchen14_MiniGPT-4_finetune_train.py - {'id': 'QXJ0aWZhY3Q6NjEzMjczODU5', 'digest': 'e1357229d7190882bf8b2dba86a37a56', 'state': 'PENDING', 'aliases': [], 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjEwOTExOTI4MQ==', 'latestArtifact': None}, 'version': 'latest'} -2023-10-23 08:57:46,226 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: defer -2023-10-23 08:57:46,226 INFO SenderThread:49933 [sender.py:send_request_defer():622] handle sender defer: 9 -2023-10-23 08:57:46,226 INFO SenderThread:49933 [dir_watcher.py:finish():365] shutting down directory watcher -2023-10-23 08:57:46,360 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: keepalive -2023-10-23 08:57:46,411 INFO SenderThread:49933 [dir_watcher.py:finish():395] scan: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files -2023-10-23 08:57:46,411 INFO SenderThread:49933 [dir_watcher.py:finish():409] scan save: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/config.yaml config.yaml -2023-10-23 08:57:46,411 INFO SenderThread:49933 [dir_watcher.py:finish():409] scan save: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/requirements.txt requirements.txt -2023-10-23 08:57:46,411 INFO SenderThread:49933 [dir_watcher.py:finish():409] scan save: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-metadata.json wandb-metadata.json -2023-10-23 08:57:46,414 INFO SenderThread:49933 [dir_watcher.py:finish():409] scan save: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/conda-environment.yaml conda-environment.yaml -2023-10-23 08:57:46,414 INFO SenderThread:49933 [dir_watcher.py:finish():409] scan save: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/output.log output.log -2023-10-23 08:57:46,416 INFO SenderThread:49933 [dir_watcher.py:finish():409] scan save: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json wandb-summary.json -2023-10-23 08:57:46,416 INFO SenderThread:49933 [sender.py:transition_state():626] send defer: 10 -2023-10-23 08:57:46,416 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: defer -2023-10-23 08:57:46,417 INFO HandlerThread:49933 [handler.py:handle_request_defer():170] handle defer: 10 -2023-10-23 08:57:46,421 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: defer -2023-10-23 08:57:46,423 INFO SenderThread:49933 [sender.py:send_request_defer():622] handle sender defer: 10 -2023-10-23 08:57:46,423 INFO SenderThread:49933 [file_pusher.py:finish():168] shutting down file pusher -2023-10-23 08:57:46,933 INFO wandb-upload_0:49933 [upload_job.py:push():137] Uploaded file /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/requirements.txt -2023-10-23 08:57:47,293 INFO wandb-upload_1:49933 [upload_job.py:push():137] Uploaded file /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/config.yaml -2023-10-23 08:57:47,381 INFO wandb-upload_4:49933 [upload_job.py:push():137] Uploaded file /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:47,442 INFO wandb-upload_2:49933 [upload_job.py:push():137] Uploaded file /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/conda-environment.yaml -2023-10-23 08:57:47,443 INFO wandb-upload_3:49933 [upload_job.py:push():137] Uploaded file /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/output.log -2023-10-23 08:57:47,643 INFO Thread-11 :49933 [sender.py:transition_state():626] send defer: 11 -2023-10-23 08:57:47,643 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: defer -2023-10-23 08:57:47,644 INFO HandlerThread:49933 [handler.py:handle_request_defer():170] handle defer: 11 -2023-10-23 08:57:47,644 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: defer -2023-10-23 08:57:47,644 INFO SenderThread:49933 [sender.py:send_request_defer():622] handle sender defer: 11 -2023-10-23 08:57:47,644 INFO SenderThread:49933 [file_pusher.py:join():173] waiting for file pusher -2023-10-23 08:57:47,644 INFO SenderThread:49933 [sender.py:transition_state():626] send defer: 12 -2023-10-23 08:57:47,644 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: defer -2023-10-23 08:57:47,644 INFO HandlerThread:49933 [handler.py:handle_request_defer():170] handle defer: 12 -2023-10-23 08:57:47,644 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: defer -2023-10-23 08:57:47,644 INFO SenderThread:49933 [sender.py:send_request_defer():622] handle sender defer: 12 -2023-10-23 08:57:47,845 INFO SenderThread:49933 [sender.py:transition_state():626] send defer: 13 -2023-10-23 08:57:47,845 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: defer -2023-10-23 08:57:47,845 INFO HandlerThread:49933 [handler.py:handle_request_defer():170] handle defer: 13 -2023-10-23 08:57:47,846 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: defer -2023-10-23 08:57:47,846 INFO SenderThread:49933 [sender.py:send_request_defer():622] handle sender defer: 13 -2023-10-23 08:57:47,846 INFO SenderThread:49933 [sender.py:transition_state():626] send defer: 14 -2023-10-23 08:57:47,846 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: defer -2023-10-23 08:57:47,846 INFO HandlerThread:49933 [handler.py:handle_request_defer():170] handle defer: 14 -2023-10-23 08:57:47,846 DEBUG SenderThread:49933 [sender.py:send():375] send: final -2023-10-23 08:57:47,846 DEBUG SenderThread:49933 [sender.py:send():375] send: footer -2023-10-23 08:57:47,846 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: defer -2023-10-23 08:57:47,846 INFO SenderThread:49933 [sender.py:send_request_defer():622] handle sender defer: 14 -2023-10-23 08:57:47,847 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: poll_exit -2023-10-23 08:57:47,847 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: poll_exit -2023-10-23 08:57:47,847 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: server_info -2023-10-23 08:57:47,847 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: server_info -2023-10-23 08:57:47,849 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: get_summary -2023-10-23 08:57:47,849 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: sampled_history -2023-10-23 08:57:48,027 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:57:48,027 INFO MainThread:49933 [wandb_run.py:_footer_history_summary_info():3476] rendering history -2023-10-23 08:57:48,027 INFO MainThread:49933 [wandb_run.py:_footer_history_summary_info():3508] rendering summary -2023-10-23 08:57:48,028 INFO MainThread:49933 [wandb_run.py:_footer_sync_info():3434] logging synced files -2023-10-23 08:57:48,028 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: shutdown -2023-10-23 08:57:48,028 INFO HandlerThread:49933 [handler.py:finish():845] shutting down handler -2023-10-23 08:57:49,027 INFO SenderThread:49933 [sender.py:finish():1550] shutting down sender -2023-10-23 08:57:49,027 INFO SenderThread:49933 [file_pusher.py:finish():168] shutting down file pusher -2023-10-23 08:57:49,027 INFO SenderThread:49933 [file_pusher.py:join():173] waiting for file pusher -2023-10-23 08:57:49,027 INFO WriterThread:49933 [datastore.py:close():298] close: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/run-bxwpu63q.wandb diff --git a/wandb/run-20231023_085438-bxwpu63q/logs/debug.log b/wandb/run-20231023_085438-bxwpu63q/logs/debug.log deleted file mode 100644 index d243e71..0000000 --- a/wandb/run-20231023_085438-bxwpu63q/logs/debug.log +++ /dev/null @@ -1,27 +0,0 @@ -2023-10-23 08:54:38,127 INFO MainThread:49300 [wandb_setup.py:_flush():76] Configure stats pid to 49300 -2023-10-23 08:54:38,127 INFO MainThread:49300 [wandb_setup.py:_flush():76] Loading settings from /home/chenj0g/.config/wandb/settings -2023-10-23 08:54:38,127 INFO MainThread:49300 [wandb_setup.py:_flush():76] Loading settings from /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/settings -2023-10-23 08:54:38,127 INFO MainThread:49300 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} -2023-10-23 08:54:38,127 INFO MainThread:49300 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'train.py', 'program': '/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/train.py'} -2023-10-23 08:54:38,127 INFO MainThread:49300 [wandb_init.py:_log_setup():507] Logging user logs to /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/logs/debug.log -2023-10-23 08:54:38,127 INFO MainThread:49300 [wandb_init.py:_log_setup():508] Logging internal logs to /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/logs/debug-internal.log -2023-10-23 08:54:38,127 INFO MainThread:49300 [wandb_init.py:init():547] calling init triggers -2023-10-23 08:54:38,127 INFO MainThread:49300 [wandb_init.py:init():554] wandb.init called with sweep_config: {} -config: {} -2023-10-23 08:54:38,127 INFO MainThread:49300 [wandb_init.py:init():595] starting backend -2023-10-23 08:54:38,127 INFO MainThread:49300 [wandb_init.py:init():599] setting up manager -2023-10-23 08:54:38,129 INFO MainThread:49300 [backend.py:_multiprocessing_setup():106] multiprocessing start_methods=fork,spawn,forkserver, using: spawn -2023-10-23 08:54:38,151 INFO MainThread:49300 [wandb_init.py:init():605] backend started and connected -2023-10-23 08:54:38,153 INFO MainThread:49300 [wandb_init.py:init():695] updated telemetry -2023-10-23 08:54:38,561 INFO MainThread:49300 [wandb_init.py:init():732] communicating run to backend with 60.0 second timeout -2023-10-23 08:54:39,256 INFO MainThread:49300 [wandb_run.py:_on_init():2176] communicating current version -2023-10-23 08:54:39,594 INFO MainThread:49300 [wandb_run.py:_on_init():2185] got version response upgrade_message: "wandb version 0.15.12 is available! To upgrade, please run:\n $ pip install wandb --upgrade" - -2023-10-23 08:54:39,594 INFO MainThread:49300 [wandb_init.py:init():782] starting run threads in backend -2023-10-23 08:54:52,772 INFO MainThread:49300 [wandb_run.py:_console_start():2157] atexit reg -2023-10-23 08:54:52,772 INFO MainThread:49300 [wandb_run.py:_redirect():2012] redirect: SettingsConsole.WRAP_RAW -2023-10-23 08:54:52,772 INFO MainThread:49300 [wandb_run.py:_redirect():2077] Wrapping output streams. -2023-10-23 08:54:52,772 INFO MainThread:49300 [wandb_run.py:_redirect():2102] Redirects installed. -2023-10-23 08:54:52,773 INFO MainThread:49300 [wandb_init.py:init():824] run started, returning control to user process -2023-10-23 08:54:52,773 INFO MainThread:49300 [wandb_watch.py:watch():51] Watching -2023-10-23 08:57:49,249 WARNING MsgRouterThr:49300 [router.py:message_loop():77] message_loop has been closed diff --git a/wandb/run-20231023_085438-bxwpu63q/run-bxwpu63q.wandb b/wandb/run-20231023_085438-bxwpu63q/run-bxwpu63q.wandb deleted file mode 100644 index 65178528f75e3becd6da742592d089831879b4ac..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 106822 zcmcef2VhiX*7uzW(HJ{+Y|-6SY+-uAU0u<&u4V0=4Gx&jJDrHO)w1rSA0 zQBV<7R1`!|RP4A)v10`rVE=yqn`AQ2b8|0pneW~GzJ1r7%$?Jo^PKWO=XN}O?@sSO zwb9*szTYjqQK%#|AhcJzvP`!wiOR(J-Oi7vQ;BqCw@f;fj(5!_v(bu7sxq0qH?+~S zo0pcBbh+f>-hET?3olyz_{*_Ss3bff6j}GT_u9O7w(`=7Oe}usg_qS`7y2^P@}zc~ z_N=O{>fXElqS#(rm2OeGap#LGYP)qA5DNe2icVWMeU|8aQLJ-)WnF#sX5D-DjYXmd zMpFmYTo>A`wcr=-Z{dYp{U88jn2jlmio;&#$VjtnXV}*|vSMYrL{+rc0M}JkljrneBFdEZ!y2 zrAxfBODqwMWX{iajU*~E-BO0&&h53JWEaF<6x*?M`_gS4UJvY9SzpnuqQ0WMx~^Ak zUC|-?T*Y6sIqSGq`Dq)F$v>oLgVyPH*8DsWN{Y`qVLd3>q$IRXfv-O;Kddo0r`0XnpEqbH5L_E00BD@dKlg17q>BNTy>l(J>i2Af1VpoqhE2N1SkO*~#@4 zed^2V`&6L9+U}+6|I;@Qt3P2WKfOuyJt2Pk0q3_Dz4qP5;dY}!?jKJakWNOLe_UFv ztE}(br=q^9S6x|cullmO-j!Xe&abNMc0gIvr*&l)SJl*%b*U`t+N-u}MSW#0|G!&V z`<@lO53H-dw5Brmhx?ZG>QmOW2dwDxiz~Y{J)!(?e&@uh-jy|cvTu0+CMS*jVG6?NL@&by;QI`fGTHZ4Tb+Xr5eJ z&DqMaKd#{ho*XJ$|L6U9$>?p5<9VA^@0jv7S&zzXk2^m2cL!8{i|oMZhdPi%WG;XdRBDiA1Z6gyI{vn z|F!XdX#CHbic2f|)Rjld>ndx`?_5z|Ut4}*=iYsKRhQTF>QmAA(DI8btGf57Uw?k* zov?DI_}$Xrch#DYp)Ux(zg#u#4)ME{!EeJypB0qfPJ#Ko{9|fcgqh2T)*F(weM*0yQ{(P>=A%7cQg2HSUD^Zzp;X!-^#E1){5WV4SrWFxFI0Fzg@EZQQ~(Gliv?-EGWOh z^;>ah_m{+PnaS^yBLnjL`<%C46Tf>J{H`51dV~2b%=vw2^<(dd-@OceS1rCJAirHM zSTIlg{>k9C-}u`K%5QM}{<+74XNccF8~o0{>W+Z?p1XU867l;NgWpMu#|Pp!Uhwlf zW5lp~#qZt*zYPyh2*~fUPu~7O{O)7$JA3Zk1?4xmehQ|CTvZ}2$s5dXVb1UK3!eCo^t+$I?~1ke1?0DVkKqT1-~A1K8%8`(P=15!H}(!E zw}Z*=RWl39Z*cu~9o_F)@f$JtUG&yt0r~yrkS98b->AXw@;hd2Fu#R4zws--UoL)Q z2EU8Hc`_is!}?!VCVt}vzq7x4x}f|9*Y79SeRZ4oO&I*n{_2^4{C*pH_8IY;H27_J z;kiKkCJTOkkKKPO$DUILzbo!~J|MrD{cd?x{H9HQpIT5*euL}xg5I@zO1~L{-+ni| z7?9rs9;*IH{ALY)C%yX82J>5(^ZQ!A=bU(>qrq>({jUV%cca*)A@O^V!SCYfuNIWw z;QD?1l5jui_pb)OEBd`2kl)GAJhDvu{>|Wb+~BtZ@tZ36`5ki5V<(BDFZ%%x_`N z@0o`k`>yoc$>4X~*UJO)d&Y0qOcTF{8T_tV^FcxR4X)pRTypgm;`eZa-+psG3drwe zKi=uo>;7TzJAdqoK>Vf)ety5XdH!PYdxXL7@ByC(?=c3yGpBwVkl%0T z)n6xmk2Uyhc>Mc<@*7;g$F=MKy!id6!SCpSs{``e>EDM<7r)0D{LX&%$3XmM3Vwcn zXn)2b;`exi-$g@y4#;ok%YXbx{GMR&JMQLR3(9YB{oZs?$7SO8M1$YapZ^w+-{D8k z_)`4-%iwp_nEqb`>6hi}w=m~-i{3;2D}GNh`CU2y`hxV!dR)HG`QrCvgWnaOUR_Xr zgX_1`tFQDGzo!`d4!>_uKz`r)@+_ymajL=Z%oW!L;x}9H^V|2>sV|7%(+qy6K5>0O zekTv?`n2?Wy20=0Cx;Z2-{AUP{lRIQir+H~ekU~y3&`(3zuI<@_&w9$chUSCHkjYS zoZp?^S^c{BJyeuL}xxxc^X*z-9Czy0pHIUv6s zZrJ3~G05-S>TY@Xt4SY?;dv*mH~uPGcmURpLx;4Ng3mJ;Zg~2(fDHe0_LA4caA$+z z#q;kdEW^PyJpZzDKM}+KHW*%X>$rdn@89i!$zr&|V0i7jcWyYtg*n07c2V=%mY&2s@6{?Adn-!6vh42I|b_&TwJQaQTlX%@D(v84ORF@>W2GuUy;q z9x?nMgW=&1E-oy?!8KgH!{;A~;mZw%$Nlt9K!)4=vg0pe_zHvJ(a$dl#&F>QS;swe z;y=Xje+`D$j$am#;V*i;x~~|%(q#CF_Y2E#a1Ebu&4o`M1A+ZQ2FFvUd>D}9T|P?x z`Bom-KV+~x`-hJ=oaMrt;)&-j>o1lEm@Lnn|7k#$cM9J)S1ey;vOMmE&kDHS{zFjO|YqGrdoz)x8a$(N$pOzf7 zqgcMqV)^#90a@<3_m+dj^7SUmt0w+bSeAop`G{%Dz7WfUO_oQ0{&PTp_3zV` zisc(kmWSVXePLM+uH|F5e|Me?`zDj+aZ3jWWcmCbmN;j5q{;Hs3B!Z2TzJS<&ou|{ zCzeNy2@~tMzi|5~3SeAop`MQQZwiC;@ znJllk@%Df$f4t@J@5S=%Cd;GOjtjpO<--M&gJ-(j-6_O&|$vi#$#J3E=w zSd-=DZ`@T_mV;}#_lz^P5X<9CmRF6P7?9=tPP_6>X?eWK^3=tXHk{?coaJ$+cjzUS z?=)GSJ>lMfEZ;S@_cXCQ!DM;uBlj1U<=|TGJbR1##PVGx%j4Eg3CQy4uh(BAmhU!M zp8DjpU@R9N%Jo8K)#GA$qRH~?hKB>P{CNDFJH+xmCd%acr& z$1Qm@Aj`8KU)okIPc~UzJLItqXSpzE`PxCp?<|(@HCb+W=kb6n&s^D~qgcMrWO>#9 zW)+s@;9CCmyg!W)%lDfskG|unfGqcD)9G}vJjG)9#b*Ms94kDkYswj&o)XIsm@N0Z zc}_r<-*~EaqF8>=WO>Et=L*Yma4mN|d`X#Do@%l@{JnVrS$_S!x3Xe+n#JYWW|xiDw>>XQ#TQ7k`hvHZ&a0PWO?S^?VRZAIg{mv2fhu)a^cZlhjxGTbFnj_mT7&((>~r z%QHu;E-cHzwS4n7n2iNlO-z)ztmKRzqzxjJWmjC|e!A@1=C6ndF*Io5h zkY;PF@Tjl(mrQo**e{zbuXyunC=4>E`r@fQPmp21VzRvG?LmcQIk=X895=3;w7ke< zdF_~nfGn@tDf^UIe$`_6#p^bl<-(lhs!X-xzFxCf9y&N6%a?Ti?IE%Jy2EYF^NcR-eJS$W9e((*Es<>3$CQ&^UR zYk5xfzrPd9%T1QY{Wd8e%Twkoe@!gEZ?e4hf%`U`<-(lh$}4(rES5hoS#J30{(vlh zws_HjV);Xp<%aJcC@jmtwS3}TvriJsADJwVetK#^mQQ`K#sN-zY_h!S%ZG!pTzK5q z=^q|iDwaPnS?)J;dO(&hKK%E2((M`4}L5l%a3g| zvQ8|2X0kl)xhFQ9<-(lhsb^5_vy7nbGVTK?p| z8=O$&OOxeEOJ)aT`S8=edrw;a%4B)Ph`GU7E^73WR7nbGVTAsDTz^qtaWwP8bVL?EakC{5Rn^^wFV)^wKH=O0doaOSD*Eng# zw-(DczZ8(=nGYTKnOOeLWVzvnR|?B=a4kP_*<}f7`FoS)(eEt^$nxWRY`dLU{=sB< z)wOQ~W4Z9SueTDjo)*iiO_m#ec{3o(f2}w?B$n5hEDyhZabZ~wuH{X3n&PzK)|xDj zTk%dnmd`r+-wuZJN0a5Xqu<+bmJ4&1```A(71HugCd&;UEeXi-(legDURwUyWO>!c z%L>bKa4oMMcbqc-_>0N%=*jN~WI1`rtr@ZWtI6{0Pd*C7a=h@kub$f-{ugQaH z(>@8v^4C|jzd|hkZn8Y{;T45tIk=V+hraxZSnd}#=9lOH_E|udKYD1+kz%=j*qC2# znDOO?vs{?7{N$E*9xIjym@H5I>8pS&|7q~uyNKnhOqTmyy{fP*2iNlSH?A8dmajHh zp7i`T0a^Zf;w%R^G0cmDK@RK6}*0)zb1cCd>0* zU0Ya|gKK&3;@$owmK#i#SKaheK$f?A_ojb{r9px z5C1J7%jH+Mcaq2JO_pbm==XJyW-BjGZ+Yi$KFh(ioT|FOseKJLSzhsGe<%#nY<*?y z)dy4{!y)17{qnYK&L1#~2kvBS*>v7n9Q0+yt8ZLWEnN*Y37`MVlK~0;V$!i^oU&24 zWLUVmeIDUce|wq-?%*c8OI5`meeZRzJ5D)hvyIx6mX>t6?6KYl_7u9vBytJ}IO;wi;y)Uir z(W^FIp6yTvSqJv6=z2j#_sY5s^}YIZ?a`rceN|1J9?|uJ-o2`7>&tsz+A$vOSaxb% zWuMb4`qWm{cJEkre6QL{NB+F6y{fi%-}<_;9u*f=@@n@B`&RX-tn1vpPer%Cozkao zWqH|={MV|=TK=T@4fgtWX<6CDjZdp9*MDih*M>iHuYCh({G)znBnP1Jo5F3cDBY~I z*wM^@P$*KmOKF>$UUeLqw$buaX+%+46Wa8jr*?$#^!CNhQ+ywa#_*m3%!EX}i&` zr+@Ll90(a1uD(9xoabqdZpS?}VM(anrKj|%sH*K))~%wxqS1OfmYp3rx2){!va+&B z2N@avP+4Emu`JRtk?5FAl$F)=;UCKRBoc}8Pw>*QtdW(nGEvpBELNV%CiqNGZT{P` zvYr)}l=ZCa*|97hk0j&k|62Cj54ihqdAd=}zM_$l(G8PeB#4W?;``z|uHu0=hui+w zOGlCNY&4RMMEQ3lo=T)LNk1KBP(db^DBV3@N86yIbRw0FX5-m7T1v$8RTOO-E(v|N z&xHN@L&@lH+sjQ#c5b$otr2l79Z95KZR$#e#BM55U!kMR^rwmcS3 zMxyc3-STy_bu3yQ$zYGzax9rayLqQ40Ud3YtXlA>L~^UINI1RmGzT%0h(wb7S8tJI z%QMkTGM!D^B8ir#_*WEpW@7PlHfoTPgp}fGlP1rXNN#h9q+PR>Y{OHskr+l6%O=vO zNIzq^l&zc01dhvNTxfBdj#zmVZtyf2{v(mj8g!(fW8;%%@Bf<4&FwztCR(0M$1<^4 zEQ*d}*=Q`~B@z*mj%MLIW}TZ@tUQ@YWAj|P=>+~SUpZ+g*<{+?7j_pVclc5gOT{zU zY$B0N#^dQE$J4{hVjkr&EHdIc!LfLGB$ed+a&i*zWSpD~l$6|Z1-yR$QMc$QZ~JM=eZ>k%D8n1C7Z!SI5n|2 zRvY&^Imt+@99I#IrETqS)pP1niAaV!ESAZXZl1e%!qEtnly z+ASy#i>XG^nRG0jM!24bN2DaVEYtj!c?*KRgoR`@nTc}x5*bV-pBVYr&Bk;uPKcO0 z{fLPo-BdOmPhlw>#x%a#(^z8V*?20G#*Mm&NoVkpoaI;|8_D2^^NEq8-TbdRR~;pD znc#0OS;d=Jr0!untv4-LMIxGx$1}E)lI5{XmNS$|M3dP}GHQ?_N4v!_Tb#B~a=B|= za^c3nX{VE^ScXX0j~3)lP~-A&NzOT!X)KMq&L*;WmVCL$+iv-fQ&;^gx!mn57sMXr zU-5V{olL}ej)&JzMr0Qdld@Hm;wntU6U0q1t}qPHAVuDGt78to>jlYWqQ6|?8IDjo zfwd$PktC;P}(6*o$25))U4I>#Z|=kMEhryCf6YB<4bh zkc>yY<&q?rNb#4pxunbC1co$%jdT3{r0KvNkfzUHAEzo*RH}T~sshzar*de`2^2rKOlI)?_^9bBt1)7H*4F z)J;q>70txs(L_8JPsEajWMXo>+x0tj_8TJRK7V2oIH)xD8J2)|$VR;-gIy44W!%w1 zww%C;$8g6Y-$cxiOiYe<`2ll|HxROEJd?7QN(3q{HUzNFwq6KUP^CBg{X6Axj6aT-A=#u;qfxsM_fjmpC1xyB+^8L zq=vrvVKPeON0Q;Huf*d-3YZ55!o3*9Qu4JEm$%)0REM1pR$eyUH!tI2#$iKC0-uoZ zzMG*WlT7J!n3*b+kP4~Rqf~?|H7|II%hB%9W81lhNjZR#Az(4>M1FR!-S_+SM3M5CFDXgB8_@-qijzY`yd5p1aJ_K8;%-uiF~|?e zr$}Td(Bzwn{Oq4z?0(3bBIR*kQm8i($l;mDS&7`e6X$56oGd*ZrQBdfadE;syonBZ zmWPpM=93~X`{!3)ynla@@`Nub7%Wv%&W(cYXx5h$QaEa%F0)OQbM_K(!a@82_o+dO z-0WW}9`3h|NSWnJ3SXxXnhn%B&EoxR25^}ZeZ(m&gKZ)gO zf3ZY#jMLa|43AED>P-qEf121gVkfI{LOZTvP7q>EDr3#pQ9}N9-xVvrpCMvq`x8Sr znZ$bWF5D!lqj=g2A%7At=c*_`42~cfp`t*YQM?(%$mQ-g@tm^XM9ee(#3<3saFubM z$v=F;>?k3BmLQ9}*)ka7JL*Oh0k~L6#8W0Qa=QDE{GsQ`GMHz54TfqM|1ZM*&N)EF z-u|Clkh9CFx4cWXJb|&0KgzcdTpD7L&plw}LxV=^NYC*-(sEu|P9vT@MFQyK{xJ)D zE1uO&3gJPT3zINGcZflXeC~lIiS1qzDRX^ENkw!5sIYUKk}-+}p4W4xT*g8UZ|Wxz z;o5eR`v@T^MM5KYEw50+qB+M$& z>y&XG4XMcIcKBe;9*blu^IWEqAKJ#UI*SQ1M7keTxjc`MEP{xD#M(5K1o3{{={Y2* z8{u6HO5}1Q2UX9BODNC#3I%^cprDgM&_P+jkCZHyNq*rbh432f%I9WPV9#^72|3(o zL)B@WM9O?$Qji9&w$Wm#_j{8<=Z7l`CvP911Q9fA^mqQ$#^YSA6!)@3bUx0Xt{5i{?_rk{5^?+= z{k9}IH7SvrjCsbJ)J>GQp!?QKWxOx>8ZUPb0RVvl#bRprK4OUy6ma2C{IHI+;$O@+ z!?C8)Yu0;`x`&eUj@qt=H1x7dL;22y@*|}!a%Q3^ZfS1?(Hc{Ch+ncOA^as)rs7G! zo#cWwgra*Wwc{JzUzbo`@f8Xg4Alf`xYTc{`*_DKw8m%_Mo4Piq$H!XWN5nLq7pPT z^Yx>9D81WlJ>L*1i~LEUtVfhg97n#Wx}MiPL~l&>d~QVR9wPeX0;4~aq8gx@v_XpQ zq0H;OF8HVP^Qucf#tEV_PN^qDO@lLpsd%+CA%&VM6<&)Hs@*C=lYWrpr8tTPCAx;P z-`DLtOq9Ilq9or^2vdkMB2>5N5mE*Ak%@8xSHhYg9a0D1DWR!CZpuX0P{+tKZJmha zbzk*R(^Cu60O+^QlXo}OejcD$(c4tk*hsqw~HzVij>8^q!0^ch*@c*(_~Hi`C4+JY+RR+ zo0OD7IPd~wkb;)vIk=R3?cX0b?G*>b@{TVlghMf1KT&+HvJh{nkRj_AUFk=HOem8T z1clljIhCm(`PxH|_-i&KxAH%iTgg`tB_lE*N@=RsaE*97I1(Y!>x6A8RJpk?=q;ip znkS~KP07`E8u(q^YU$@)AE6L{P$0lXB3m8>P{CU$ScH1h;LWVFOqrXOU(#_0KwXSL zJ0)Lx*h$^Lzg(oe=R*px95QdwkyM(9<@%9APnwI$F3BZT=f>Z2jb+K;&1*R&XM1?% zOS7FS@)CbiD5sOsQ7pxL$*sIe$s!d@f|{r)6-c4P2^G-7)t)pWgp{1^KlVQOf$6Fu zFLhOscWOrWY=u;an;?SN3G9U*9l}9ym~A>Jb5kh7pAb@UaTwZ3%hMk5O8T4^q@88H zBH`CFxDrBJ?m?x)UY=I1Fj7O8drMO0rgluMLgW-v)RTXZ)AF=OzS?28r*(Rk`<|XC zxerdIxgg+qyM|#QyoOf2QEtTJxY{Qj2RNZgUkV~=dB(L zg47*@v}6(|UsiKd%}3%UqlTQ8n?2^*z9$|lN9TZ7;L0GIH0o`w=7UTme^ zkc`tGQ@a6B1MbF5m(%jH$Da3Nc8w_c$cGX{PScRU0Dq0_V?NOv-6yVX!Luxx(D=Xu z#MGS0P}Vl>L|*ovf1h$pN|b!;qQtNhx^zNc(Jx4GxqBXC2oZ#Wi_T36BBdNC09Kl( z7-vUL_PBYQ_FAKq@Du-(P$(smOImDTL#cLp(}F?JG)&NVGNbEcQn3NY2Jlu|0kTPq z-0blsZ}0o0#PX@HScHvX)QmEKr6@Gk7A z3r1PlU8_yf?jUPYt4{+GS);6YCa=ie{>SfKVG?{#hQh4gXXgDd#^sGq~2(d0H`8 zhHR|)!>Iis69SRu4`Y5G#_bQmMMvAUYy8cG`C%Ip)!a`e&CiNKCPmWCKbf*V#B0Zr z?RIW_YTEp;h`L4dM;YrQfb`9e%$gtR_tKJbTM37MFr?E)a}c=?E$OtOc9uc~jo)-j zhp;#IRZBc=G^COH&=wEWwoqir2a57OwB*wUpmQ=uktLsE`pT)6Pc0D@Q`e5lNE*4Z zq{OdAlW5cVkZy?~q$1f!^JkWfjL$3)IiEHCAoXM8nV4)$CerwWwt)D-7HTX38Bew4 z!x!T@QZ4xyq*~H39%{?Rd1w=<5E6@#!!=Bxk&n(-+o zD3WPbG=FJ)SS*K~Q#28f`DrXAT{hF}mQXxrnB0iQSFwUj&ZPMf^Fu=nelnsp?fPcA zSw?h=jSrDgjvl&k#%FDmDd)aui6~#42}?rx>ZCDXJ~W>XD)l}e+Tw{2vgF9I#KTYK zv`O{SerhZySL#*vhdi}OU6gL@4;^Re9FwFio_tQo* zE}XUf%~&GGi6tKnd>$um0r6oIv0U}WW07nu_n9RhN9xVrv&7>VLGx#pbezw2EZv&@ z0A2NK<@G)KR919jbT}MOw);!zj?M>+)b{FgX?ev(6;(BwSwF7RfY7E1O^<2zr`=Yi z8#n$1H!wF|yxs1l+vI<*PhFk9QQ}ND(Dmg!Z=3&l?v9qvPqy2mbo=hk{oVM`Lu+b! zb*-qc>^2}&tRbvuyM0P`(;w&&m34Jo^j?3sTjP_W?msA^%-L>-(#`XKi05;2#L|$z zU2D(JeZF1e_nIF|{DMx~ZCko&+4bK#sK5#)wTOm=}zVYTl=GS zrOp$g<@p~Lr@#oOn<RnzlZK8=#fCig#Lhmoon ztaeol^0P0B-D!q8D92EKpcU#__`@unnMOOBrtuG)iW0uyy|e<109I-GxhD<0bD`3} zHLf%;KO>E+z(wnqCIQ7zx+h+FprBJiySmEvDR43>UaEXgJtu3DqBQ-ymv>9PEmGF{ zlR{=d!GMMjU3wY?ex!hS24QOFfsjHc4&00to&p+;{jh^bf}0%A?w5ePCAq&)rKV@|woGm-L>KPj|q(K9K7gR%Fn z{BbYSmD2CINuflBk$~5vRBV>HvdYsd4xZg{l}P#7pA=?`sCm;%qs&XG*jp-?1Whj7 zrcH`Y87)%O46Gfbupt#C>RoD%`ml{k2EX{Gqr#qokO8%;xd9*jXx#!`YcQ-N&NrlV8#`d$-p@kHZ{h0ew5VdUl4XrRvwNl-;J2rOuUmU6z?lg> zmba%O*0QHUSd~1S8FuT9Goz9fU{A#kSC$=hG>rTnu5O#h$LJTP9S{g+_Fdi17pP>= zugGVLT4$8`UM8AA7~`XSO0*~m$Q5+~m%V@yr?iTpsEUHmG_|DqMdgKe{&0y5rhk#& zh6tQ3Ey@_YTu; zZel3LbF~5*amHdyVssl-ANEUdrl~SK8I#o`skcN}%ie>?e zP+shdV1WQ;W0IoV=z^Uq&n%W)uJM-(m|R>gr%_FRfR5fuO0%JY$uQRhP6})`Jy>E1 z%pV|-Wih&sYC7EAaE|2C;F3#z7^QeYLltU@X&Tl0B++2eD4h+0NU^I3+$e{TaXLT{ zDN`=Gka{lPt#3cc<=S<~MGHX~O@p2|p2oXvnMiZ-sFUup7nbsXf}q!)k=4-v&p%1d z{)*ZU*1UC>h`G*>m?)_PxLo`T04ur$KGUcPZRP>}`)^BhTAw z=j)HClaj9YRT4)UG=(!N(m2)X8P&0y2iw#Xo*}++ zA>Njk63%WN*8qJMB1jh?0#XR32w*s7V1>CR47tej_KEb}ax2MYsJ~pat4rxQidOf_ zTP{>#1f$|dEoYZj{T0=L_F~?IHl|68Ja64LkKT8f>}8lsF8R(*Yb!XD1a=UE{JqQt zQox@BKC*irkb+_uLIgMncnnevxybL<@3r(oXC1+Cf4Oi3a1M0uaSve3y}@S`pm5S$ z>a7r?rBWntn$6>;G>MVt?K}GHF}q7iH~15y-WJ#y!L6Ka8s3t@Fo>0et2D32G=)KW zM4FVs%dl(5Bu1Y1qUX=q=U-CN2$z!bJul7@L=k}<83_l$`$|GET6IB3%|6ru(I5~` zBg7H~8A_7py?EQF8>Wkt8+}Njza?`6+5lSKf87anKwLa(OxvV@qR~7l*?(_WnQ z6_=E?pHnGPZt@{T0WY~A^58In$s<+sDlk9_VJ*wbYz2XtP}i4>O~Y`&!tRf%UUuGqfA^7G zM)}Hx@nhyGcml|}B=`q!xu{wMkjZt5l8jYjxu}@v1Du2@7kSTi=XEAsTBzi7rZ5yn?-;9w2pqt_(`#bgwwSVev$X5AobHtu?=mmmxiFl~1E{cp2GCHyw_Io+;m0US zxoL4SCp-dPmq`OdN13)oCH+DtKfK#%BIY(XG5J>`^9UShVKD_f@P<4kl?M>jxgZ;9 z^wxKlE$7HY!g~eEvMGdI{ z14^U`xGD|*n$$$0=Bm)7N1s_B`yK1DUqelRAhkn+)eR*2-dPEaAgXedI^CoYYsl;3 ze#mf4^Nq(K)q^ZsCehsO7EOMr zNh*N9qXI%=BGx>E2dKdjk>1&&QQSvH4cJ<9{iM1EG4jCIg)ZB6iz$U3wz&(|S2VUyeQ#B_=WQz}JVSZZveh^mLEEo*1@MBtyigX(1no zCCDo>I~BBS(GdHQhLX{7rBg67O_&82MJ0nnH5dK;9yz^9zNbldHW`P|!6ZB0As)1l zvT1F!q>P( z#rzgEG*Y#Kfzv4PsC7{$WQPp1X~tkWxFqFqBR%MEbMFu$sy9}0LJZhV$; zo)U7xH-zqc>7uVBnkoLGA-ob&nK`B=WUZ%Yz!l(6RHCy)LuFnO1WjGqaCV)QO&3Ka zBSO0@xy=b_9`L6Ize)Om+JU)iiq2b6?0F*4Ag^~*LtTeNUu7o7nan#OAuoJm=%BWP zACRIR^j8!MoWX?CE}&kkNqcWaVI4RWx8tKUPlK7ri@=jB!#dyc!Z(G^z2$<_rKqX? zqT#xQF+zXNH<7TfXaICm^mZu<%pdhcEmfp&tV!1VJAsuKMI|Fc&lL~;o2Z%QPmKn& zKx7kwa`lphc`Hh*eF%FfrdnpB(M4w8R43)CA}P(MCMh>ODzx$Z*C)<_num&-*XFNw zx{-^fEWO%y?FF!1yB(S8m{0??5I}&#C+mYNFp6y5sJ(5}fwHNnq^W5oLd+~5l z)4Qk|8JWH40~iUi_F}K?=MI#j9`RQcGv`P(B15+y2JEO#Joci69wd8) z)##2I9XkH3doGY@9`zFq@h44Dtq@?afN;%c!Dy;+8h|5Xi>7fJfRn(SAB`q6{b%__ zQOPZ#9{>KhOfZ8PzGV^CO~8^6^qhyr`mRwH*@}BW@1S5L3lNOU(_{fOkvJe+Dm@ID zTT0UXG$!eIp2eGv%#R=(r$-`pQA{n^E}6(aE~>X%2-? za4KwM>nMEcy8@a!A6g(rAnF3rHGOEI7|jnY7-Mma83jR7oyS@r&o`^HVv%DZo49L%x`U@Smeo4?zDYPX`W{{8&H(5F?}u8pWAg77PRp@t1$>`} zO1Fe#JhnCPeNG{*f$wui#0KB%u`M53*3>I(4;Z%n#hjm{0AYNlFXkL9_oVTmb7ljg@2Lz@It7ZoWefVg2p~?o-C70TZ=xx2 z#ydH!+qRCq=eCRaVeio`SpPqOy>I-(^~TDw^@7rX!e{?&WAawqT z0R`u?KY+XkFp~C0-Zy^gLf$hyCW~$jc~9g@q_z&^ebXO#An(;nmbVpKF62F7X*RJ= zqFx4A(-lW+Do}6D0QgP{VpS`X zs%G2~`t7I_+K8Gb{iz`X1lUc+0xn6tM$f1oYBY&W-NKd)(G>&XAPqq(l4F<^xumKY zV?%Fzy5qN^<|%(_gy$d;WH_A7ko+HSY8dCGGDWM`riSrc_Jyk>;*^Tayp-i>MJ3}x zpS-uyd5PK6zC~sBH)|+>;eBeuKBa%4P=K*0JGn_gH*f^Bi9WlP3bH({sAPQT(E~f( zEy2uQhhPB134SKv7#WmzR|Nc#T5XiEZEC1tQcKo~Z6N8S{YLV|{Jz9U35p>Sln$*TJPu;Z?*1e?*a1ASsI!RWK<(@LfSs%o;8m)QU>(3XR?EsO2JM zjxQ--281isazQe7XIZFGI>^0i4pJyOyYpiGv$cXH_5g$FvtrupmeFl?e>yARcE ztq%7o!ZDSHLsK!{O%2OiaLSB{sNQ6@I@5B!_l7S0=8`EA&4P7^hNG(0d)(W;tHU4$ zTG(P`w;!@X6wtX~F$WDsTDOL1*r!%ha$l&p>(IZ8m>1l{lJRO9Qd1vzaOKx~tXnwlODB1BMw;i`!P>@4&kuVKt5TPTNvZt;v5Uy?Q zvoYfge7M08}(+(a^#603RUepZh-9gRkt0t5JM@GYP4QDrO_)*Q+g32IFbe; z!7A9I(Kxs^xw31G!5uOP!;a;O9}YF_d+6&D&1?Rm;YCq;K{66XO`I676Rw;ZKq;6K zH!*~qTBgTs3uC79%g7Ty5?c7d^raHb>u%BHJ3o-0pxYQX&;nr4@!qnbR0`;wX1YsJ zKnoBJjbQF7vJ68sEL|%qnI78y!(~o_@Pvl_XrkG&0a?I( zL8>Alb1+S6s75A9da+T=J%yntZbXZ9+?g%t>7NZ0`k;gcxa`m?!G)K}Y?$ z{YGtgYk7oyKiDyD2fd+pOGNU?GeZBq^76}d&foGo=X#GDS)-_G9?WU9*Ci{8YEu(z&YHN$#u-$GqFe)0(NIYS*;= zf=AprmD`kJS=xj&&CT1BU2H`qkA*%Qb8xY!S?o^@C|SifctD~HAb#G$0i492p%9lf zgFyz_iye|=;*~(T_GAp#(**AY9C-|P)@of2OoH4~9Ww30LEDTY(4ycBm0^*O?+Vbe6H}MG;&R=9ywI zTT#hVp*?q5u(ilp;!h668rrbHb15HbxZYb+%&Ic^ASA1O#Odp6)CmhGWYs?DeCe>9 zt*GSb(9 zWs7g7ikxMBgY6L+B~s!ge3h&dH|ngYFbrA#cy8!l@7;Hs$obHp9I{tz zSmQN}>uT4CCpln0nER#ZXOp8%d76<^`alF{m6Nh^$j^nI-*m^`a>yU8!y#*fFvL)@ zqf$mz>3MZxA83`*t7SM)Lt>?^lw9K!Z!N=-L!K8p>iG1%63)kd!ohGDMx+Em97x_N zq`s$c$RCKWh^}1ZaLEw&Yn&A6+2i`G;3z72J~ZIu9=%1*Cw}DcAOX&WY2_Ni!~2rM z-HIu@Vs>6F%IG$zH-O!EQ!`8k0|YtqL;Kx5HzMTQr#a-?kWq^#Z@l&LSxV?o>77iM(Gya8JIEFi> zoKHhZ0AfmjKG5|nChrWhw;~)eXl*+XI9uz3^Bf>n4K*)@p56GFU358o>32D3MJm}n zQwDm0fbV_SCu*2{{fZ2^mO0B5<3Y@+32o%DYSO#BT;GUYqzHI%~-SP)B)MI1i%=- z?&bcVMyt{_NMln&+|1~P9I5hHvtq%lC)B(g`tGG6M~Rx1{?y=$Njuf*qUObPtPeG~ zeX=>X_hYP+o(c66QW+97<8GjG05z|K{<_a;n~9oLZfaVr7|T?{sc|KN2I{=4Iav*X z11)zAaB?Sb`ZyY5wbzV${tvw;CXLC4STMs z>)Lq?#0W>Ion*RSEMjy|y&BrT&((`1ns42r$u}Ek@Q1!JvSyowcZx?xj29%4EW0mp zI^Eip4ILyP0=MTM@Tjh-*FvYh{I3-{Q{VZWDZHP;EQ)}{kcfs-jHlgT9dvV1hRcfC zup6hDuDZLNpOKE^z#-@L(2PqaB}C5me&kT1Aw#2zqu7lr+$Z45VjaStl2Y1-iV08Z zWS~>YUFocwM?5NL{6=Wi@NX{@IY0Q5LkUn#CM~ORYL4F00ZgXeI~{5_IgCW%7a>Po z1M^fx<&EDAJ-h!4BSg+>e{$%|<0(iT*hN9D)W_w3?FZ1Uky@J^ZFb}qV~+%0i?)Gj zJMzbGg%(`B$1IVv#-AMF#hnptmTD2>`W$jBgz`F$w-s_Ykjwy4$Dyofme;9YK+fBt z9d~?wq{vz8PY%_24h6ZM%ItDqo~8_@mH?XDs1woTyfM~>z-RA8}{#T2k6fV4S**<4#X^2qOme%}80sd76%x!q2_ zM^@FAyn#v;3n8d0`jMjo0Lw~TPLRqwWfRp9^(5@OU-W4+2z zjaLnJdQ(H7K6NGOK=k-7QPi=IgB)IzPI3;;3advXMs229{_6;P}$yY zFMNDFhwmjtHBgv9G@+*Xs}_EXuH z0Xd&}gX>!`$1y(v!LmV3BO15)v6g`NFz>OJeDq-sa||HXek>n01J+xUPSa1ej<0VL zx{a@QmP+UB%LdrXG!rdocMCzzAVo4~L-vCEHUvy9&$UEU%o}d9IlHwH_8QP=`LGok z`#fI4P)Et7^M%Xv%vl%IG2`Jz8Y>2(_nI#_M z2U{_qoQ<$g&*0yr#x0!OyBu0a*f#^(-N&oti`Mbt7xVVeTj#5hoFOd;ti_rlm`GJgPL&mVbU?Ac4^T%WCC?BOdNTPMc8@vpcI^bGfQdm3c1 zd3Sc~s+jtBznEHWE;1>tNTbxIUW0c`?QE1(CBqfsQ&c2D08_?R5MpYxvJg`$zch5l zTl1VP9R1q(>||1*Lu;mZQ#(;Um0r)cpi(xa%?0ey9pMAo0GUZ;LhnKYW@*OM%H)@Y z!k;{Ol_LE9e$=SVpcI_S7)3;08|c}|5jDt;I+#l~gqm6`NsWwjRf|0GQ`MLf`Q@Sa z54z|xQ8U1w8n)`Ga;t^XNShX*M_-BZ1K|l!30EbP_Yw)NNBcIacoSOLA{leL={p1RI`v7Qx5-8XyHqz>>|Mo z@)HavN{tur0-zb4t&ZM$!ZPOyDwaa#V8W{fVxZ+d~K?XX-v2ejqPG3gqi4t{nc8(Dku1obpeDA34gt zDVIVIwL_}T{^a1`0ByLXL(v&nIt$>KJK=ezvf@hOKMf68pqT)sE}z< zFzbK3r9)nfmDAL9DT|__%5=C2U~W{{OzFrQuLzABwDERQ)^+~msPaRn0Sp9qMz%)y zk%J$geC9q>+-IOpu^MMfiHU)ryaYL)g?6o*JwS)*de@=KFUjL(sH9WY0n!bc)7$-F zA9R)IEt?Y~42iI9QgA(u_gEEYXA9itq2H%<*;0ZT>@OG^9rVhD(x$nm3Z+L86DSny z3tR6N3`+&MHz-5lj5Shb>Pf!%i_p(|oct$|GsK@90di>}s?VTn%Ck-f?iNSMl@I>h zKI0j(R&9C09pNZ=hHM=v=EghgU+6U$C)y7zj zLF*Vq7Kj!8(}3&YSs~}E(3ii~?4|LZVQu{0;()bqhvWXV9TYqPoVPmO!*)#2Au0*l zMLO+F(Q#rsle0?Skc^!1*P(7tTu`nS*YGy3w_6w{jJxMDq{+(qAn)-WkphwhFp8_X zOF;oog>TS&g|+{Z!7hkd8Cr7rOFK(VH~6awX1N)7hO=qR`=!}LKMr(=)gZRjupvNg zXWp3&E*BFtosF{#ZdGW}#ak{HIV1eZp?O2G60{_nMl@pPZ8ch(0^xuFEF*>-t;Pf- z#fO?FFpM}MSNu)rwjnS4plItx|7Z)upn_cu%F!O6VeWavF%R8ol*cW@fgHR!OM^7c z#a0=U8hPSxLkAx4+bN>vCVy(U?*uq#e3ygbOdo2v82K@m8BzQBY58LO{pQX@zF zU1;f2LtOA_*0g%RdzYMoSrbv?3XGu!kg}*@LJ}ux23rXQ#@~ms z^A^T+sz$j_RZG)^&>NNGXS6UDKk5@0s}RPtA>y+&#rP$cB|H8ZDPVfel#P7x51}Kf zkEoWWZgy)b|A^DU!TX^uS^_>V83M0CKLZPMLr$sg(e`5S6yzaW$gGPhAy>RQbbt2U z=S9tEH#Pa90qfzss+`^nFcqIrl!~?Xh`GUER1?`T#<9|ykGSwmPwXs&TNA47H`&1r z-r{FB*dKSV;2gAGv?|N{o`M<&6XV7dK@RsWh9?LZ;Ud4vR!Uld`q5(}G;!|3)^7z+Lj>{e3zHgwUJ=EbW@b5rIyG%Zp7_Vm z+iSMFPX>Ie+ko?JhJD_I2%swI9ukRq9&TvC*0Em~^1)f2L=MERIIdh1!Hv^^n4dyZ zy1swDAaS>~X*E((Dr&~LsmV8DHJgdDxETnEy>|dW zvjYMpTX4xndpSura1Jy@iTaEq<*bAIHFUy}(aEA_ygxN0PtIF$$P?-H`Wz_|DPi#F z{M*3j`dZ@-;>7gA*ZOL(DjKZ!2akCqN4Z z$Xv2fs4PGxs0RvLR+5?271b}?uym&fC7Zk4)EKg%^UYp4pg^P@nlbP^R7$i#SYW00 z^-Z`;iHVU30&ColX*Wt1`iGA>XJAZ6{BHjvP8C4q36RbhF%gb8HRRdEQN&~};gCwP zBo|#NXe0SG$uU#nNO!OBHc`lhe(Hcb{nY14{&Ry@$81x2flmsr6D2sxf zNsaEQtHQ-&`g|tg+~Y4Cz+)iLgc$(ZsddYqsv^^tqHu5asUlj&(SR3J0kHhDEmDp2BZpg2J8@efN9!jT{7jKfQI;~vkvkq3e&x!{ z)gouIKRI}OFqGOi!=cgUcTee%Pf;G^7HNeXL?a_+gadbNMmNs3k3r#GZ&^J-LB+j( zK?OlNSqI3wrrB8UYB3Fa&FHyjjp)5nf+Sq#auLGVbb!vbk88pQjXJVUc6^_|9b?3x z7xk(e0)A3A&*BYe5RC!3QC3i+RUfMB2xr9flwmk>#|`0~PdxB7Y3hDIP3b`zebUn;e=jddHHExbAbybmWn*55NB9d9TQH9(0?Iaa-{V!q(x9)kpE8CIe^^ zOKXK1&NlY~XHMf&7BzCngTp%x{OLs%lcu_gNoHZ69v8<-FWvK=d!n?%{RlqH?Ts7X zEsJnTHCX~;ks@zAB)r$u{U=FJ)BN>>8zrg`rjCMu-WlR$Hc$fyQ~*4eWC#q2Y}l+X zn66$FoPVgC-5*24kH>Czmc&2g)>HmHF0?C;p`1y@gONbbBTg4eX*RO4t%h-GpmbEe z$mW6Eo8He^{xK|k_Rr^DEUS6gj~cb7)STk|E_x}b_QiM#M_@2YpYDlab}izGQ59th zA`7!9kXCFwJpA+Y@kdEjkNB&KP9@hS%?$cN6pFm-#q^+Pi>N;65)P3eL{Q13(#sgM z>2RF=A2)>e-}G-wbu~U)DkWOMMo`J6(#R#;#5NTvUXBYC{;*C^o(+++)v< zPQ7@hpQ7}#FJfXs6N;=red!Z=1sszaIpR^_5obKNi>P_rO^tCa z$Sf&75}4pjykkGQh2*x14_%6)1A{@)*+g5;h?}7(`Qn?yCtoyio=(&g>rfPVFVEop zlU(4G`1!5Y%Sqr2>A<+h-T{H(g!np*M7XlXdYs)Kqr){nc1p_I%<}U#z=?>iRYIp{ zq2R>3k`6H(aeR~8hRvn$1UOEt6%#vE8Jn?PesW%mUF1wk%4szJKfFIUs9$lP12b`2a*{HjA4J1*k zJ~Xxg`ku7HBuB3J_VAKbUtJ_}p7A4xTMGTK4Fw%=U~tRcsv>6q5ujNJOFEFl1?B8h z(LxF9p32A<-w}Rqvrf*t>z;L6PQJsTfhfE`RS$|V-e`YRO1_PKFx6w3G4C}cx?$>x zbeySo^9Ikz7mp2(I-$oALi^8Y(}?z;?v!X7ru3VzZnQs<6aG-YB9x=GN57O(4uwl# zkFh_7_Gi^&4qb0yNzFz17MRrh&;pYd1NMzNi*tdzmRm?$7SKB|RA&;v0&8m$fVK8b zV*(bIw3w*c*`4fs)5hAWP}4jK;Kq`IU~ZZcu%Vx>sPflymP`!{aU z6_4&k`jlKVIrzLSA5}-Q%>Z%!!T8mtR|z!kytf2oAYWap^qVBx_)S|t01}fqf-DIY z)5MJD7O<0@8w3>-%cli&bRPVFv`fDQs@%#h{k#ub5l~K_t-$^BK>N0U)(!V>mVhN6 z2Or*KB?!`xodGy*9q!-sR2%NkQ*|;M7v@_A1Y$oCYgVRvn?4y<4h8N4`|~b5Z?Hcw z?*jW%D$3#6*L$CS?wjk2_HQA`M*B;+EswQj zulb9X1sPwoPMArmWk7t`GDWezcz?a;#4(aq7V5X06CltoyUdBMHN1bzhn9F6MbP}4 zmX8*jxseYW@Q-|QX312Ei=N!MwHN9)KDFs5TfzJ1q4upi%uz{`x@@?AzPfC*f8#F( zhx=nUyt-z+;Qm632Z#H|Q-5fqzFxt`V*l+M5#>EAd-m#c>7g|>y}DM^S9Ti^;_8XC z+o5#xM$mfW#{)veoOMPEcQ4&e|9cC%-uk@tjrum+pXgtDZXNFLeE3J;{&ejeCuS1V z4EJ|Fb?wv#hNfR`9q&)IYTY~aoj>xx`=fBjxEn>+yVu2)eNL?G)3vg;zVTfFDW(b9 zndG+iM?Pr(RQ!cbpO012JlCBx=NEod^akg`j=;wEw0efrTvqfMDOR~)M1+?@jj4jf z23QN2mr)8oF8t3a9os3uf6gz!r+-axgqnr6*@C_CfrCJe+F&e*(T96qBQeth_>&x;m_oO%A_khrj}g!Ge0pC+t#`U5%OX=q}(igY+ywpVBeK4>;4 zXUQkW*%o(ac>E@NI2&)C_a_H1933JO8@&OHo|(6<2>7)~oz|Y6{(#+K83grMfaENY zH|s6Vwzvu5bAJE(5=m#iKRG~v_z{XXY_Ow%<6Te{vI~MW65u9>N;4H+%J^~8F|(rT zY>T@qyvbevaW-NsSO+;I&Dz#Nmyv(_DNE2Ux&rRm4DL1>3Uo~X0MW^{Q1)46@^^<{ zdh0O<{P2R`!4edMmO8*2XZx^UMU_~Nk2RU%$_}+#U!7IJ(o`4BQhZj4{KW8PJG6Cb zV=wwqqnsDZ0U(980EhVGs-UUp{ApIE6>6OG4F=5tQyS+gt33Xm@anZ=_m^e|W`6J|Fm%`c{cEIU>jKPkNN(AxcF$1nLAj)GW> zm*-IT*1H+J%gqGOLP$`9aM>~A(HefCC`+e;uB0g(CGwNQr=E1&L<#3*f8pqjx^!4+ z__7FqZkxAoSeU7o^SNsrOna&FOM4qL#}sg<9wlv)q+I8qiY&N6Ru1{T@Rqx*eqYoqa#LeeTsTh5fdN}&uE+az z6(R-%9feM}KV}&s*M`~xJO;AzWkYQVYVHq*XDoYHxVTqyxVR^rqRY6+BR=+hNh%Mp zSH&ee3hkEnlGH@YsyMAH$;ZVdQiaCF-S^Yc{}V3mwPsvgBO{aMj)IXOxVUZ;7uHHq zue%l15?0U&SBs7A4efM#Cp61FPdJ-Ig*j?XQCtiZsPryny_nfllyDeR!gEfU^s7p| zZ@3F}Mvs>JK)nIx%DDWzuT5e>mdB{KW#3a;VE_byy&T{}g^A6(#X+1v%md+`y(U~M zck|{t+zlu+$EV?J=y3Ux!=Z83SGviecCKjvjf!CHCONvN9tNf&zPYiJFC`AE{t^(!fT|Amb@t&T~QB*f0*+6XqoW;{7e{}G)ONE<8Us4kW=&b z>idxlGB`wy*%kR|#$D$dcm^Jwo}EdK?x;t?hh2Gw(-wHwpB%Ln$UtN{NACyF9Cjb;J?@)x87=Fm9cx}>Iu4|wgtCnTNs{K(+YR>0l8sx|q&j?TN{rkTpou%s_N0|qnM&STifp;rX@GcHAKBX0M z2p!2+)3gwsaWaD(dE=Sk_Io_@KasP{pB#1UY0xUB04wFa`k(lj#dQz^Qq7hQAv2&T zL<}}qli{Xxt z`K!*HDQZ6Sr-qP)_wFzV1{jiJgZCZ)p}VwF)0K&**eD@L5(gZY!%{y}HuA?$hUdh7 z9i}_{BljJie{XWW;R}^>Hi2JzkC#J@G6R0hz8om_fl>h3NahPv)g(sV_^I$M<0m20pO=lC6D}E+?>n1ZkmuOb_iH4hj zD1agxR}kd|Xz~^f*1=8WZtkZ*?=V)WHzT4d6?hEM$P+&sKIy>U|08NX^QT5D{D3qO zg>p6-Kk( zY#8tk>eTx>tkH5rLvWAP5gTfOrXd(P;kn@%8?DL8X1;LSO#bQC;11d#$V_+qeXeljt9&Qo_75!Xup1dfQ zxekg@+T%>jW$H;@`1$Z9Tc3HCi22%|7$)r$>cC{`84^7Ame;>F0%o6LjWHg*yc6$FQOgF`WJ+@Dy87&TDJAweb!r)-mgC z*s#3tg7Cc!KRCN#R{4>`rJ#+WdQS=BVQAhvCI;B0fk#TPt&jt>SrtQI(i+Gy$Z?i@ zyb!+WrD#t{=NmtAsE#|+MPTbVFVx$-A><&tF!CrvXfARD)}jv8v_!?6f(5aMbY2YK z``1fLM9#N<RPLa)nxYcP#)SN)udoq(kPt5SJ%j7|tfn zo+Cs4&d-oF48eRGWpD=zsYzI`QzdK{cxd;_jP&wa>;{kH^kIx142J_^UJ9osob60l ze(xt6YQ56I00mQ52|MdD=EkB~R;H)rbk^B4E-*6~F5}mN9K~-a)7mJWaadFb|aF3rt z4Lv2Ww`|~PgcRo+jdTRIpBqfr>EOFGO76Z)}^h186! z!AyM<^2dwA7f*S^dGp@yZcA=CS4f9EUW*^tH>1_te$*(<;YwC`&Yhjkj_=VVxMJ&YI^`}C)~%o<4yf;PQgy-U%IL{X{yOu8_0kWkpQmLv>Z9) z|AkkLZ!ER-FZNl;M}CJn5Q8(BM)g^%6FpBkIi;pk+>K4hLAt>GDUo65;2R7>mP39w zyz5_v{Y}Cd;3pi;6@CJaX^|ps!Z>eoun+QEf;^Yg0oP5l2nmrO+HIJG9P)eNHxC~2 zxTv|xpBg+d(>zo~DKR?G1kbBeo0lk4xDAJ119zU(s78TR!AZy=FA4u`=eM`gsk(X{ zsv>ry>a4;Y`y{~7c&bXH+K?`?dGh#0M$ZKp0%bwTR1SVhT#%9B^1K&OxMg|pEx6R{Y z#eko83)I(KcEH7ND&V@N*tM>!<>CZ%L}5!S|5%tw3gmx>5%S{){mBZ-0!xU4gIkS;e zJ1tC;(KWR^eD9!bwwDE8>u5;{CE=3miaSo$%MV_6?04#tX@6iG0xNdrH5yUP);PpwU@3?@NnOA% zZeCgh5amxUtAN9&MAG9FZ1I$7+NzR^y;K;^O&DsJ$W)M^ih0G;1zcs;{rqS>Uuc~PN z)RIs!`eSaW`BU43*vG?nn~Ac7#ES@G$=rvnNU7zU_9=5v@=cFjzi|lQYPqS9*2W=n z=ZH>>Atl53S{;XIrpFRd;}@Ghv?SDM4Bqn)8p(2o6YSR?V758aqylD>+G0elj6>v2 zHCQg%u+3KHA@XJuEb%z>G|_60Ll~b~(#huoRGiI6%L_#8VF*ag zWNzl8m0<{;+Vn;H4ttmIL>&Ghc63X&D0!qEv7(9?!$aGf4eGner{>Ms%`Gft#d_e%o z7U;lIPtx?c7d0Gkl6|(n*l9EJBR$X>aUf1~2h1>E6^;+XSHJk`4%HAdq_{dxSp9|* zR?i&s!{kuAKb_L2qHATBimn%wweQ)huD*=7g)ysA*0rLhrmUu-u72N+rR%@4amrs_ zn)MHUaA;aS6qtFOpCYwE)O_lDZ_EGs*rs;07RuMSmR zDlh54L{fES*ZK}!Rnq9tv#PeLd++*K=ku#-E9?8#R(3d!|K-RNPdPBr@;}x0si>+g z?|tcB2b9(DXJz0+4=AguE&FTFUfufER32Pf#=m;>ZGF{j7e>-=nsH@Dg z6-tthj2xLbLUR6F=eqie`ns~emDN?&oZl$0&Xv7;b?wo!QnyVcjf-827k)4Miy5;*do8Rzl}hCcl@8p54cy0{2B$*Xlk6+KDABatEjuc zkYRvRSiH+75lL=TzT}qo8yQ4$yPvq{6Cd%cn~SUOZ;@QTn@1zLTMH<;dibvEAcO0y z|C(oYbjQ~zwa$%(SfwM|xu#dI-t7a3t-MQp4~cE`|Lg9|LgXmIIG#>+*0}MA3o*Ki zE~Cg~b*JapIaXZ7sJKxhKB%C}GIUSZPIpgV(^cJb_!1Yvhyg`>h>1r`H13*64x>i# zBt!y=#0z~A4-j8OK?%VNzv}6k>8@Q?Xf=p0GrLpM)z!cGzN)Y5U-fkm+x~mEMZtD- zS^UOj9JlhZn0l;-ZS%p$0oyC51RG2~dDs@9@l1|pi4J=#Cdh&TQ9{~;_j<)J7IaHP zc*1{=;=^xlU;7P3x-E$GyDd*fL0Wxn(?wrFUE5>od=Kf|(H($v>nTB6NCga$@(QB# zpd~u2rs9-vd^&;|zVyRc70T@KAhR7?cSd1$@_0k0$8J z)KrBR454fql91(*;bI;&5w9sCZ#9W*8gNI$V}+nm3j!#if-p}^yDamD!M6hl?7s&c z7#3M}v}be2-k$@0hk7>mkMOeu8wencmlRBFMRZ_edXOUQP4FoCygY=a zLoA1M9JvI;cS(}0lK+*&h+;tT-{iUKhI6MV&mBRYhfmH&;W>9-;Jlrj4c)Lqd%x zzCXWab{l2AGsybk-7iF8{n-W2{Y-s-SIGCj+_&$4?E9HthcM;$OpxE^pY})L_fY?> z3XK_`4aJOeTV4cy|76VQ4II#Iw(s89m>V86w7~1VINP!j^cad!DIr}%k_3NsN1q6z z&i5NcFWTN6B)5P6p(y0u-L<_q0ABuFO#P_maG@au0xf&E8v z^~$@Tu01g|-kpN@{L{C9_2A8@DVh+tY%mD2ZO0-Jyza>TC!VKm-5YG{caI*8($?P^ zsRI{7T`$CDxz+JAa&o%bYUFSsQ^yJJ5w0H|ps@N7_tS$PBu<3WLv85;WvVM2E>M?2xa-!ssJiT51*mqTq~$x!3$TmwTU zbR26j)I9y;SYRK{9Nu+GIS#{UJ`6nlN3N_O=*Hp3pAu>0$RfXw{#kaeJY z6-$s|)9ZzSQ~ZU^^Kwq4O=l zWf2{@*On!?urhM+e6XxP%q7rHRz!y5y57m=)C-ldeN3>zB0FNTZ0aS|G3d-ydaSf@f!fuc`-H$XGmPld)B%wM zJMCd1V@8}QiWx~1ig_Vh5OVoR?l$i3ehsn#Vj@1a99ChjtqX=Ej$#N;A@CoyOid{_ zN8R+$nt?O@4{s5gFatB#7r!$TkGlhFZB%m<8|e@(;pB{Mwvdq|4KjFM<>vC_qD;_4 z)yK!`7T4OuWiz^77&8q4NkS^;<|Xxe@6IGO=qe(`(Wufsg3p>PU!Afhh zSw>A#MzEA|6NhxoZRCZ5qT9NoYqo>xV$o9bQ#2qRYgQs~|O#lr}zY*0L7K zH=U*?Rg0FR*2W5EPO)vlsuxMEJOeMeYQ9icwFbsTT__7;D(jl5vSw%9Dk@~0vRpMC o)heW0Vy*tm^c1%WeCYs$+shN&PHx#3-`;jdKYVg9c92{4CuD^O<^TWy diff --git a/wandb/run-20231023_090058-141dzr9c/files/conda-environment.yaml b/wandb/run-20231023_090058-141dzr9c/files/conda-environment.yaml deleted file mode 100644 index cfced9e..0000000 --- a/wandb/run-20231023_090058-141dzr9c/files/conda-environment.yaml +++ /dev/null @@ -1,297 +0,0 @@ -name: eye -channels: - - pytorch - - defaults -dependencies: - - _libgcc_mutex=0.1=main - - _openmp_mutex=5.1=1_gnu - - blas=1.0=mkl - - brotlipy=0.7.0=py39h27cfd23_1003 - - bzip2=1.0.8=h7b6447c_0 - - ca-certificates=2023.01.10=h06a4308_0 - - certifi=2022.12.7=py39h06a4308_0 - - cffi=1.15.1=py39h5eee18b_3 - - charset-normalizer=2.0.4=pyhd3eb1b0_0 - - cryptography=39.0.1=py39h9ce1e76_0 - - cudatoolkit=11.3.1=h2bc3f7f_2 - - ffmpeg=4.3=hf484d3e_0 - - flit-core=3.8.0=py39h06a4308_0 - - freetype=2.12.1=h4a9f257_0 - - giflib=5.2.1=h5eee18b_3 - - gmp=6.2.1=h295c915_3 - - gnutls=3.6.15=he1e5248_0 - - idna=3.4=py39h06a4308_0 - - intel-openmp=2021.4.0=h06a4308_3561 - - jpeg=9e=h5eee18b_1 - - lame=3.100=h7b6447c_0 - - lcms2=2.12=h3be6417_0 - - ld_impl_linux-64=2.38=h1181459_1 - - lerc=3.0=h295c915_0 - - libdeflate=1.17=h5eee18b_0 - - libffi=3.4.2=h6a678d5_6 - - libgcc-ng=11.2.0=h1234567_1 - - libgomp=11.2.0=h1234567_1 - - libiconv=1.16=h7f8727e_2 - - libidn2=2.3.2=h7f8727e_0 - - libpng=1.6.39=h5eee18b_0 - - libstdcxx-ng=11.2.0=h1234567_1 - - libtasn1=4.19.0=h5eee18b_0 - - libtiff=4.5.0=h6a678d5_2 - - libunistring=0.9.10=h27cfd23_0 - - libwebp=1.2.4=h11a3e52_1 - - libwebp-base=1.2.4=h5eee18b_1 - - lz4-c=1.9.4=h6a678d5_0 - - mkl=2021.4.0=h06a4308_640 - - mkl-service=2.4.0=py39h7f8727e_0 - - mkl_fft=1.3.1=py39hd3c417c_0 - - mkl_random=1.2.2=py39h51133e4_0 - - ncurses=6.4=h6a678d5_0 - - nettle=3.7.3=hbbd107a_1 - - numpy=1.23.5=py39h14f4228_0 - - numpy-base=1.23.5=py39h31eccc5_0 - - openh264=2.1.1=h4ff587b_0 - - openssl=1.1.1t=h7f8727e_0 - - pillow=9.4.0=py39h6a678d5_0 - - pip=23.0.1=py39h06a4308_0 - - pycparser=2.21=pyhd3eb1b0_0 - - pyopenssl=23.0.0=py39h06a4308_0 - - pysocks=1.7.1=py39h06a4308_0 - - python=3.9.16=h7a1cb2a_2 - - pytorch-mutex=1.0=cuda - - readline=8.2=h5eee18b_0 - - requests=2.28.1=py39h06a4308_1 - - setuptools=65.6.3=py39h06a4308_0 - - six=1.16.0=pyhd3eb1b0_1 - - sqlite=3.41.1=h5eee18b_0 - - tk=8.6.12=h1ccaba5_0 - - torchaudio=0.12.1=py39_cu113 - - torchvision=0.13.1=py39_cu113 - - typing_extensions=4.4.0=py39h06a4308_0 - - urllib3=1.26.15=py39h06a4308_0 - - wheel=0.38.4=py39h06a4308_0 - - xz=5.2.10=h5eee18b_1 - - zlib=1.2.13=h5eee18b_0 - - zstd=1.5.4=hc292b87_0 - - pip: - - accelerate==0.20.3 - - aiofiles==23.1.0 - - aiohttp==3.8.4 - - aiosignal==1.3.1 - - altair==4.2.2 - - antlr4-python3-runtime==4.9.3 - - anyio==3.6.2 - - appdirs==1.4.4 - - argon2-cffi==21.3.0 - - argon2-cffi-bindings==21.2.0 - - arrow==1.2.3 - - asttokens==2.2.1 - - async-timeout==4.0.2 - - attrs==22.2.0 - - backcall==0.2.0 - - beautifulsoup4==4.12.2 - - bitsandbytes==0.37.0 - - bleach==6.0.0 - - blis==0.7.9 - - braceexpand==0.1.7 - - catalogue==2.0.8 - - cchardet==2.1.7 - - chardet==5.1.0 - - click==8.1.3 - - cloudpickle==2.2.1 - - cmake==3.26.1 - - comm==0.1.3 - - confection==0.0.4 - - contourpy==1.0.7 - - cycler==0.11.0 - - cymem==2.0.7 - - datasets==2.13.1 - - debugpy==1.6.7 - - decorator==5.1.1 - - decord==0.6.0 - - defusedxml==0.7.1 - - dill==0.3.6 - - docker-pycreds==0.4.0 - - einops==0.6.0 - - entrypoints==0.4 - - executing==1.2.0 - - fairscale==0.4.13 - - fastapi==0.95.0 - - fastjsonschema==2.16.3 - - ffmpy==0.3.0 - - filelock==3.9.0 - - fire==0.5.0 - - fonttools==4.38.0 - - fqdn==1.5.1 - - frozenlist==1.3.3 - - fsspec==2023.4.0 - - gitdb==4.0.10 - - gitpython==3.1.31 - - gradio==3.24.1 - - gradio-client==0.0.8 - - h11==0.14.0 - - httpcore==0.16.3 - - httpx==0.23.3 - - huggingface-hub==0.16.4 - - imageio==2.31.1 - - importlib-metadata==6.2.0 - - importlib-resources==5.12.0 - - iopath==0.1.10 - - ipykernel==6.22.0 - - ipython==8.12.0 - - ipython-genutils==0.2.0 - - isoduration==20.11.0 - - jedi==0.18.2 - - jinja2==3.1.2 - - joblib==1.2.0 - - jsonpointer==2.3 - - jsonschema==4.17.3 - - jupyter-client==8.1.0 - - jupyter-core==5.3.0 - - jupyter-events==0.6.3 - - jupyter-server==2.5.0 - - jupyter-server-terminals==0.4.4 - - jupyterlab-pygments==0.2.2 - - kiwisolver==1.4.4 - - langcodes==3.3.0 - - lazy-loader==0.2 - - linkify-it-py==2.0.0 - - lit==16.0.0 - - llvmlite==0.39.1 - - loralib==0.1.1 - - markdown-it-py==2.2.0 - - markupsafe==2.1.2 - - matplotlib==3.7.0 - - matplotlib-inline==0.1.6 - - mdit-py-plugins==0.3.3 - - mdurl==0.1.2 - - mistune==2.0.5 - - mpmath==1.3.0 - - multidict==6.0.4 - - multiprocess==0.70.14 - - murmurhash==1.0.9 - - nbclassic==0.5.5 - - nbclient==0.7.3 - - nbconvert==7.3.0 - - nbformat==5.8.0 - - nest-asyncio==1.5.6 - - networkx==3.1 - - nltk==3.8.1 - - notebook==6.5.4 - - notebook-shim==0.2.2 - - numba==0.56.4 - - nvidia-cublas-cu11==11.10.3.66 - - nvidia-cuda-cupti-cu11==11.7.101 - - nvidia-cuda-nvrtc-cu11==11.7.99 - - nvidia-cuda-runtime-cu11==11.7.99 - - nvidia-cudnn-cu11==8.5.0.96 - - nvidia-cufft-cu11==10.9.0.58 - - nvidia-curand-cu11==10.2.10.91 - - nvidia-cusolver-cu11==11.4.0.1 - - nvidia-cusparse-cu11==11.7.4.91 - - nvidia-nccl-cu11==2.14.3 - - nvidia-nvtx-cu11==11.7.91 - - omegaconf==2.3.0 - - openai==0.27.0 - - opencv-python==4.7.0.72 - - orjson==3.8.9 - - packaging==23.0 - - pandas==2.0.0 - - pandocfilters==1.5.0 - - parso==0.8.3 - - pathlib==1.0.1 - - pathtools==0.1.2 - - pathy==0.10.1 - - peft==0.2.0 - - pexpect==4.8.0 - - pickleshare==0.7.5 - - platformdirs==3.2.0 - - policies==0.4.2 - - portalocker==2.7.0 - - preshed==3.0.8 - - progressbar2==4.2.0 - - prometheus-client==0.16.0 - - prompt-toolkit==3.0.38 - - protobuf==4.22.1 - - psutil==5.9.4 - - ptyprocess==0.7.0 - - pure-eval==0.2.2 - - pyarrow==12.0.1 - - pycocoevalcap==1.2 - - pycocotools==2.0.6 - - pycryptodome==3.17 - - pydantic==1.10.7 - - pydub==0.25.1 - - pygments==2.14.0 - - pynndescent==0.5.8 - - pyparsing==3.0.9 - - pyrsistent==0.19.3 - - python-dateutil==2.8.2 - - python-json-logger==2.0.7 - - python-multipart==0.0.6 - - python-utils==3.7.0 - - pytz==2023.3 - - pywavelets==1.4.1 - - pyyaml==6.0 - - pyzmq==25.0.2 - - regex==2022.10.31 - - rfc3339-validator==0.1.4 - - rfc3986==1.5.0 - - rfc3986-validator==0.1.1 - - safetensors==0.3.1 - - scikit-image==0.21.0 - - scikit-learn==1.2.2 - - scipy==1.10.1 - - seaborn==0.12.2 - - semantic-version==2.10.0 - - send2trash==1.8.0 - - sentence-transformers==2.2.2 - - sentencepiece==0.1.97 - - sentry-sdk==1.19.1 - - setproctitle==1.3.2 - - smart-open==6.3.0 - - smmap==5.0.0 - - sniffio==1.3.0 - - soupsieve==2.4 - - spacy==3.5.1 - - spacy-legacy==3.0.12 - - spacy-loggers==1.0.4 - - srsly==2.4.6 - - stack-data==0.6.2 - - starlette==0.26.1 - - submitit==1.4.5 - - sympy==1.11.1 - - tenacity==8.2.2 - - termcolor==2.3.0 - - terminado==0.17.1 - - thinc==8.1.9 - - threadpoolctl==3.1.0 - - tifffile==2023.4.12 - - timm==0.6.13 - - tinycss2==1.2.1 - - tokenizers==0.13.3 - - toolz==0.12.0 - - torch==2.0.0 - - tornado==6.2 - - tqdm==4.64.1 - - traitlets==5.9.0 - - transformers==4.31.0 - - triton==2.0.0 - - typer==0.7.0 - - tzdata==2023.3 - - uc-micro-py==1.0.1 - - umap-learn==0.5.3 - - uri-template==1.2.0 - - uvicorn==0.21.1 - - visual-genome==1.1.1 - - wandb==0.14.2 - - wasabi==1.1.1 - - wcwidth==0.2.6 - - webcolors==1.13 - - webdataset==0.2.48 - - webencodings==0.5.1 - - websocket-client==1.5.1 - - websockets==11.0.1 - - xxhash==3.2.0 - - yarl==1.8.2 - - zipp==3.14.0 -prefix: /home/chenj0g/anaconda3/envs/eye diff --git a/wandb/run-20231023_090058-141dzr9c/files/config.yaml b/wandb/run-20231023_090058-141dzr9c/files/config.yaml deleted file mode 100644 index 5c53ff8..0000000 --- a/wandb/run-20231023_090058-141dzr9c/files/config.yaml +++ /dev/null @@ -1,38 +0,0 @@ -wandb_version: 1 - -_wandb: - desc: null - value: - python_version: 3.9.16 - cli_version: 0.14.2 - framework: huggingface - huggingface_version: 4.31.0 - is_jupyter_run: false - is_kaggle_kernel: false - start_time: 1698040858.125822 - t: - 1: - - 1 - - 11 - - 41 - - 49 - - 55 - - 63 - - 71 - 2: - - 1 - - 11 - - 41 - - 49 - - 55 - - 63 - - 71 - 3: - - 1 - - 13 - - 23 - 4: 3.9.16 - 5: 0.14.2 - 6: 4.31.0 - 8: - - 5 diff --git a/wandb/run-20231023_090058-141dzr9c/files/output.log b/wandb/run-20231023_090058-141dzr9c/files/output.log deleted file mode 100644 index 70f57c3..0000000 --- a/wandb/run-20231023_090058-141dzr9c/files/output.log +++ /dev/null @@ -1,164 +0,0 @@ - -2023-10-23 09:01:04,885 [INFO] Start training -batch sizes [[2]] -module.llama_model.base_model.model.model.layers.0.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.0.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.0.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.0.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.1.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.1.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.1.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.1.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.2.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.2.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.2.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.2.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.3.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.3.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.3.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.3.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.4.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.4.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.4.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.4.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.5.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.5.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.5.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.5.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.6.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.6.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.6.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.6.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.7.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.7.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.7.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.7.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.8.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.8.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.8.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.8.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.9.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.9.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.9.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.9.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.10.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.10.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.10.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.10.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.11.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.11.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.11.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.11.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.12.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.12.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.12.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.12.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.13.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.13.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.13.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.13.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.14.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.14.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.14.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.14.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.15.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.15.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.15.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.15.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.16.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.16.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.16.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.16.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.17.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.17.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.17.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.17.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.18.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.18.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.18.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.18.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.19.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.19.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.19.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.19.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.20.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.20.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.20.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.20.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.21.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.21.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.21.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.21.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.22.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.22.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.22.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.22.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.23.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.23.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.23.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.23.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.24.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.24.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.24.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.24.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.25.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.25.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.25.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.25.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.26.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.26.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.26.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.26.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.27.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.27.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.27.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.27.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.28.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.28.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.28.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.28.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.29.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.29.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.29.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.29.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.30.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.30.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.30.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.30.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.31.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.31.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.31.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.31.self_attn.v_proj.lora_B.weight -module.llama_proj.weight -module.llama_proj.bias -2023-10-23 09:01:07,002 [INFO] dataset_ratios not specified, datasets will be concatenated (map-style datasets) or chained (webdataset.DataPipeline). -2023-10-23 09:01:07,002 [INFO] Loaded 12171 records for train split from the dataset. -2023-10-23 09:01:07,020 [INFO] number of trainable parameters: 56627200 -2023-10-23 09:01:07,021 [INFO] Start training epoch 0, 1000 iters per inner epoch. -Train: data epoch: [0] [ 0/1000] eta: 0:45:28 lr: 0.000001 loss: 1.3049 time: 2.7288 data: 0.0000 max mem: 33055 -/home/chenj0g/anaconda3/envs/eye/lib/python3.9/site-packages/torch/utils/checkpoint.py:31: UserWarning: None of the inputs have requires_grad=True. Gradients will be None - warnings.warn("None of the inputs have requires_grad=True. Gradients will be None") -Train: data epoch: [0] [ 50/1000] eta: 0:07:58 lr: 0.000001 loss: 1.2327 time: 0.4508 data: 0.0000 max mem: 38295 -Train: data epoch: [0] [ 100/1000] eta: 0:07:10 lr: 0.000002 loss: 1.1049 time: 0.4457 data: 0.0000 max mem: 39575 -Train: data epoch: [0] [ 150/1000] eta: 0:06:42 lr: 0.000002 loss: 1.2879 time: 0.4630 data: 0.0000 max mem: 40211 -Train: data epoch: [0] [ 200/1000] eta: 0:06:18 lr: 0.000003 loss: 1.0077 time: 0.4527 data: 0.0000 max mem: 40211 -Train: data epoch: [0] [ 250/1000] eta: 0:05:52 lr: 0.000003 loss: 1.4790 time: 0.4626 data: 0.0000 max mem: 41450 -Train: data epoch: [0] [ 300/1000] eta: 0:05:27 lr: 0.000004 loss: 1.2864 time: 0.4639 data: 0.0000 max mem: 41450 -Traceback (most recent call last): - File "/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/train.py", line 118, in - main() - File "/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/train.py", line 114, in main - runner.train() - File "/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/minigpt4/runners/runner_base.py", line 377, in train - train_stats = self.train_epoch(cur_epoch) - File "/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/minigpt4/runners/runner_base.py", line 437, in train_epoch - return self.task.train_epoch( - File "/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/minigpt4/tasks/base_task.py", line 116, in train_epoch - return self._train_inner_loop( - File "/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/minigpt4/tasks/base_task.py", line 225, in _train_inner_loop - scaler.scale(loss).backward() - File "/home/chenj0g/anaconda3/envs/eye/lib/python3.9/site-packages/torch/_tensor.py", line 487, in backward - torch.autograd.backward( - File "/home/chenj0g/anaconda3/envs/eye/lib/python3.9/site-packages/torch/autograd/__init__.py", line 200, in backward - Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass -KeyboardInterrupt \ No newline at end of file diff --git a/wandb/run-20231023_090058-141dzr9c/files/requirements.txt b/wandb/run-20231023_090058-141dzr9c/files/requirements.txt deleted file mode 100644 index 75bc393..0000000 --- a/wandb/run-20231023_090058-141dzr9c/files/requirements.txt +++ /dev/null @@ -1,246 +0,0 @@ -accelerate==0.20.3 -aiofiles==23.2.1 -aiohttp==3.8.4 -aiosignal==1.3.1 -altair==5.1.2 -antlr4-python3-runtime==4.9.3 -anyio==3.6.2 -appdirs==1.4.4 -argon2-cffi-bindings==21.2.0 -argon2-cffi==21.3.0 -arrow==1.2.3 -asttokens==2.2.1 -async-timeout==4.0.2 -attrs==22.2.0 -backcall==0.2.0 -beautifulsoup4==4.12.2 -bitsandbytes==0.37.0 -bleach==6.0.0 -blis==0.7.9 -braceexpand==0.1.7 -brotlipy==0.7.0 -catalogue==2.0.8 -cchardet==2.1.7 -certifi==2022.12.7 -cffi==1.15.1 -chardet==5.1.0 -charset-normalizer==2.0.4 -click==8.1.3 -cloudpickle==2.2.1 -cmake==3.26.1 -comm==0.1.3 -confection==0.0.4 -contourpy==1.0.7 -cryptography==39.0.1 -cycler==0.11.0 -cymem==2.0.7 -datasets==2.13.1 -debugpy==1.6.7 -decorator==5.1.1 -decord==0.6.0 -defusedxml==0.7.1 -dill==0.3.6 -docker-pycreds==0.4.0 -einops==0.6.0 -entrypoints==0.4 -executing==1.2.0 -fairscale==0.4.13 -fastapi==0.95.0 -fastjsonschema==2.16.3 -ffmpy==0.3.1 -filelock==3.9.0 -fire==0.5.0 -flit-core==3.8.0 -fonttools==4.38.0 -fqdn==1.5.1 -frozenlist==1.3.3 -fsspec==2023.9.2 -gitdb==4.0.10 -gitpython==3.1.31 -gradio-client==0.6.0 -gradio==3.24.1 -h11==0.14.0 -httpcore==0.18.0 -httpx==0.25.0 -huggingface-hub==0.18.0 -idna==3.4 -imageio==2.31.1 -importlib-metadata==6.2.0 -importlib-resources==5.12.0 -iopath==0.1.10 -ipykernel==6.22.0 -ipython-genutils==0.2.0 -ipython==8.12.0 -isoduration==20.11.0 -jedi==0.18.2 -jinja2==3.1.2 -joblib==1.2.0 -jsonpointer==2.3 -jsonschema==4.17.3 -jupyter-client==8.1.0 -jupyter-core==5.3.0 -jupyter-events==0.6.3 -jupyter-server-terminals==0.4.4 -jupyter-server==2.5.0 -jupyterlab-pygments==0.2.2 -kiwisolver==1.4.4 -langcodes==3.3.0 -lazy-loader==0.2 -linkify-it-py==2.0.2 -lit==16.0.0 -llvmlite==0.39.1 -loralib==0.1.1 -markdown-it-py==2.2.0 -markupsafe==2.1.2 -matplotlib-inline==0.1.6 -matplotlib==3.7.0 -mdit-py-plugins==0.3.3 -mdurl==0.1.2 -mistune==2.0.5 -mkl-fft==1.3.1 -mkl-random==1.2.2 -mkl-service==2.4.0 -mpmath==1.3.0 -multidict==6.0.4 -multiprocess==0.70.14 -murmurhash==1.0.9 -nbclassic==0.5.5 -nbclient==0.7.3 -nbconvert==7.3.0 -nbformat==5.8.0 -nest-asyncio==1.5.6 -networkx==3.1 -nltk==3.8.1 -notebook-shim==0.2.2 -notebook==6.5.4 -numba==0.56.4 -numpy==1.23.5 -nvidia-cublas-cu11==11.10.3.66 -nvidia-cuda-cupti-cu11==11.7.101 -nvidia-cuda-nvrtc-cu11==11.7.99 -nvidia-cuda-runtime-cu11==11.7.99 -nvidia-cudnn-cu11==8.5.0.96 -nvidia-cufft-cu11==10.9.0.58 -nvidia-curand-cu11==10.2.10.91 -nvidia-cusolver-cu11==11.4.0.1 -nvidia-cusparse-cu11==11.7.4.91 -nvidia-nccl-cu11==2.14.3 -nvidia-nvtx-cu11==11.7.91 -omegaconf==2.3.0 -openai==0.27.0 -opencv-python==4.7.0.72 -orjson==3.9.8 -packaging==23.0 -pandas==2.0.0 -pandocfilters==1.5.0 -parso==0.8.3 -pathlib==1.0.1 -pathtools==0.1.2 -pathy==0.10.1 -peft==0.2.0 -pexpect==4.8.0 -pickleshare==0.7.5 -pillow==9.4.0 -pip==23.0.1 -platformdirs==3.2.0 -policies==0.4.2 -portalocker==2.7.0 -preshed==3.0.8 -progressbar2==4.2.0 -prometheus-client==0.16.0 -prompt-toolkit==3.0.38 -protobuf==4.22.1 -psutil==5.9.4 -ptyprocess==0.7.0 -pure-eval==0.2.2 -pyarrow==12.0.1 -pycocoevalcap==1.2 -pycocotools==2.0.6 -pycparser==2.21 -pycryptodome==3.17 -pydantic==1.10.7 -pydub==0.25.1 -pygments==2.14.0 -pynndescent==0.5.8 -pyopenssl==23.0.0 -pyparsing==3.0.9 -pyrsistent==0.19.3 -pysocks==1.7.1 -python-dateutil==2.8.2 -python-json-logger==2.0.7 -python-multipart==0.0.6 -python-utils==3.7.0 -pytz==2023.3 -pywavelets==1.4.1 -pyyaml==6.0 -pyzmq==25.0.2 -regex==2022.10.31 -requests==2.28.1 -rfc3339-validator==0.1.4 -rfc3986-validator==0.1.1 -rfc3986==1.5.0 -rpe-index==1.2.0 -safetensors==0.3.1 -scikit-image==0.21.0 -scikit-learn==1.2.2 -scipy==1.10.1 -seaborn==0.12.2 -semantic-version==2.10.0 -send2trash==1.8.0 -sentence-transformers==2.2.2 -sentencepiece==0.1.97 -sentry-sdk==1.19.1 -setproctitle==1.3.2 -setuptools==65.6.3 -six==1.16.0 -smart-open==6.3.0 -smmap==5.0.0 -sniffio==1.3.0 -soupsieve==2.4 -spacy-legacy==3.0.12 -spacy-loggers==1.0.4 -spacy==3.5.1 -srsly==2.4.6 -stack-data==0.6.2 -starlette==0.26.1 -submitit==1.4.5 -sympy==1.11.1 -tenacity==8.2.2 -termcolor==2.3.0 -terminado==0.17.1 -thinc==8.1.9 -threadpoolctl==3.1.0 -tifffile==2023.4.12 -timm==0.6.13 -tinycss2==1.2.1 -tokenizers==0.13.3 -toolz==0.12.0 -torch==2.0.0 -torchaudio==0.12.1 -torchvision==0.13.1 -tornado==6.2 -tqdm==4.64.1 -traitlets==5.9.0 -transformers==4.31.0 -triton==2.0.0 -typer==0.7.0 -typing-extensions==4.4.0 -tzdata==2023.3 -uc-micro-py==1.0.2 -umap-learn==0.5.3 -uri-template==1.2.0 -urllib3==1.26.15 -uvicorn==0.21.1 -visual-genome==1.1.1 -wandb==0.14.2 -wasabi==1.1.1 -wcwidth==0.2.6 -webcolors==1.13 -webdataset==0.2.48 -webencodings==0.5.1 -websocket-client==1.5.1 -websockets==11.0.1 -wheel==0.38.4 -xxhash==3.2.0 -yarl==1.8.2 -zipp==3.14.0 \ No newline at end of file diff --git a/wandb/run-20231023_090058-141dzr9c/files/wandb-metadata.json b/wandb/run-20231023_090058-141dzr9c/files/wandb-metadata.json deleted file mode 100644 index f7f0519..0000000 --- a/wandb/run-20231023_090058-141dzr9c/files/wandb-metadata.json +++ /dev/null @@ -1,374 +0,0 @@ -{ - "os": "Linux-3.10.0-1160.76.1.el7.x86_64-x86_64-with-glibc2.17", - "python": "3.9.16", - "heartbeatAt": "2023-10-23T06:00:59.460778", - "startedAt": "2023-10-23T06:00:58.084251", - "docker": null, - "cuda": null, - "args": [ - "--job_name", - "minigpt4_v2_test", - "--cfg-path", - "train_configs/minigpt_v2_finetune.yaml" - ], - "state": "running", - "program": "/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/train.py", - "codePath": "train.py", - "git": { - "remote": "https://github.com/junchen14/MiniGPT-4_finetune", - "commit": "5c3ec8bb730b2e9df23b4bb3eb24108f9c04a8d6" - }, - "email": "jun.chen@kaust.edu.sa", - "root": "/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune", - "host": "gpu201-16-l", - "username": "chenj0g", - "executable": "/home/chenj0g/anaconda3/envs/eye/bin/python", - "cpu_count": 64, - "cpu_count_logical": 64, - "cpu_freq": { - "current": 1996.2770000000016, - "min": 0.0, - "max": 0.0 - }, - "cpu_freq_per_core": [ - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - } - ], - "disk": { - "total": 251.85773849487305, - "used": 7.894733428955078 - }, - "gpu": "NVIDIA A100-SXM4-80GB", - "gpu_count": 2, - "gpu_devices": [ - { - "name": "NVIDIA A100-SXM4-80GB", - "memory_total": 85899345920 - }, - { - "name": "NVIDIA A100-SXM4-80GB", - "memory_total": 85899345920 - } - ], - "memory": { - "total": 503.7154769897461 - } -} diff --git a/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json b/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json deleted file mode 100644 index f3fca1a..0000000 --- a/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json +++ /dev/null @@ -1 +0,0 @@ -{"epoch": 0, "loss": 1.0476144552230835, "_timestamp": 1698041023.1460156, "_runtime": 165.02019357681274, "_step": 334, "_wandb": {"runtime": 164}} \ No newline at end of file diff --git a/wandb/run-20231023_090058-141dzr9c/logs/debug-internal.log b/wandb/run-20231023_090058-141dzr9c/logs/debug-internal.log deleted file mode 100644 index d524e8d..0000000 --- a/wandb/run-20231023_090058-141dzr9c/logs/debug-internal.log +++ /dev/null @@ -1,1701 +0,0 @@ -2023-10-23 09:00:58,125 INFO StreamThr :51707 [internal.py:wandb_internal():86] W&B internal server running at pid: 51707, started at: 2023-10-23 09:00:58.125014 -2023-10-23 09:00:58,126 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status -2023-10-23 09:00:58,127 INFO WriterThread:51707 [datastore.py:open_for_write():85] open: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/run-141dzr9c.wandb -2023-10-23 09:00:58,129 DEBUG SenderThread:51707 [sender.py:send():375] send: header -2023-10-23 09:00:58,534 DEBUG SenderThread:51707 [sender.py:send():375] send: run -2023-10-23 09:00:59,117 INFO SenderThread:51707 [dir_watcher.py:__init__():219] watching files in: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files -2023-10-23 09:00:59,117 INFO SenderThread:51707 [sender.py:_start_run_threads():1124] run started: 141dzr9c with start time 1698040858.125822 -2023-10-23 09:00:59,117 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:00:59,118 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:00:59,119 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: check_version -2023-10-23 09:00:59,119 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: check_version -2023-10-23 09:00:59,439 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: run_start -2023-10-23 09:00:59,444 DEBUG HandlerThread:51707 [system_info.py:__init__():31] System info init -2023-10-23 09:00:59,444 DEBUG HandlerThread:51707 [system_info.py:__init__():46] System info init done -2023-10-23 09:00:59,444 INFO HandlerThread:51707 [system_monitor.py:start():181] Starting system monitor -2023-10-23 09:00:59,444 INFO SystemMonitor:51707 [system_monitor.py:_start():145] Starting system asset monitoring threads -2023-10-23 09:00:59,444 INFO HandlerThread:51707 [system_monitor.py:probe():201] Collecting system info -2023-10-23 09:00:59,445 INFO SystemMonitor:51707 [interfaces.py:start():190] Started cpu monitoring -2023-10-23 09:00:59,445 INFO SystemMonitor:51707 [interfaces.py:start():190] Started disk monitoring -2023-10-23 09:00:59,447 INFO SystemMonitor:51707 [interfaces.py:start():190] Started gpu monitoring -2023-10-23 09:00:59,447 INFO SystemMonitor:51707 [interfaces.py:start():190] Started memory monitoring -2023-10-23 09:00:59,447 INFO SystemMonitor:51707 [interfaces.py:start():190] Started network monitoring -2023-10-23 09:00:59,460 DEBUG HandlerThread:51707 [system_info.py:probe():195] Probing system -2023-10-23 09:00:59,466 DEBUG HandlerThread:51707 [system_info.py:_probe_git():180] Probing git -2023-10-23 09:00:59,479 DEBUG HandlerThread:51707 [system_info.py:_probe_git():188] Probing git done -2023-10-23 09:00:59,479 DEBUG HandlerThread:51707 [system_info.py:probe():240] Probing system done -2023-10-23 09:00:59,479 DEBUG HandlerThread:51707 [system_monitor.py:probe():210] {'os': 'Linux-3.10.0-1160.76.1.el7.x86_64-x86_64-with-glibc2.17', 'python': '3.9.16', 'heartbeatAt': '2023-10-23T06:00:59.460778', 'startedAt': '2023-10-23T06:00:58.084251', 'docker': None, 'cuda': None, 'args': ('--job_name', 'minigpt4_v2_test', '--cfg-path', 'train_configs/minigpt_v2_finetune.yaml'), 'state': 'running', 'program': '/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/train.py', 'codePath': 'train.py', 'git': {'remote': 'https://github.com/junchen14/MiniGPT-4_finetune', 'commit': '5c3ec8bb730b2e9df23b4bb3eb24108f9c04a8d6'}, 'email': 'jun.chen@kaust.edu.sa', 'root': '/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune', 'host': 'gpu201-16-l', 'username': 'chenj0g', 'executable': '/home/chenj0g/anaconda3/envs/eye/bin/python', 'cpu_count': 64, 'cpu_count_logical': 64, 'cpu_freq': {'current': 1996.2770000000016, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}], 'disk': {'total': 251.85773849487305, 'used': 7.894733428955078}, 'gpu': 'NVIDIA A100-SXM4-80GB', 'gpu_count': 2, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}], 'memory': {'total': 503.7154769897461}} -2023-10-23 09:00:59,479 INFO HandlerThread:51707 [system_monitor.py:probe():211] Finished collecting system info -2023-10-23 09:00:59,479 INFO HandlerThread:51707 [system_monitor.py:probe():214] Publishing system info -2023-10-23 09:00:59,479 DEBUG HandlerThread:51707 [system_info.py:_save_pip():51] Saving list of pip packages installed into the current environment -2023-10-23 09:00:59,480 DEBUG HandlerThread:51707 [system_info.py:_save_pip():67] Saving pip packages done -2023-10-23 09:00:59,480 DEBUG HandlerThread:51707 [system_info.py:_save_conda():74] Saving list of conda packages installed into the current environment -2023-10-23 09:01:00,119 INFO Thread-12 :51707 [dir_watcher.py:_on_file_created():278] file/dir created: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:00,119 INFO Thread-12 :51707 [dir_watcher.py:_on_file_created():278] file/dir created: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/requirements.txt -2023-10-23 09:01:00,119 INFO Thread-12 :51707 [dir_watcher.py:_on_file_created():278] file/dir created: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/conda-environment.yaml -2023-10-23 09:01:04,866 DEBUG HandlerThread:51707 [system_info.py:_save_conda():86] Saving conda packages done -2023-10-23 09:01:04,867 INFO HandlerThread:51707 [system_monitor.py:probe():216] Finished publishing system info -2023-10-23 09:01:04,873 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:01:04,873 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: keepalive -2023-10-23 09:01:04,873 DEBUG SenderThread:51707 [sender.py:send():375] send: files -2023-10-23 09:01:04,873 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-metadata.json with policy now -2023-10-23 09:01:04,879 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 09:01:04,879 DEBUG SenderThread:51707 [sender.py:send():375] send: telemetry -2023-10-23 09:01:04,879 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: stop_status -2023-10-23 09:01:05,122 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/conda-environment.yaml -2023-10-23 09:01:05,122 INFO Thread-12 :51707 [dir_watcher.py:_on_file_created():278] file/dir created: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-metadata.json -2023-10-23 09:01:05,281 DEBUG SenderThread:51707 [sender.py:send():375] send: telemetry -2023-10-23 09:01:05,669 INFO wandb-upload_0:51707 [upload_job.py:push():137] Uploaded file /tmp/tmpkvnc73e8wandb/uumvtbka-wandb-metadata.json -2023-10-23 09:01:06,122 INFO Thread-12 :51707 [dir_watcher.py:_on_file_created():278] file/dir created: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/output.log -2023-10-23 09:01:08,124 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/output.log -2023-10-23 09:01:09,063 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:01:09,124 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/output.log -2023-10-23 09:01:09,750 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:09,751 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:09,751 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:09,752 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:10,125 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:10,125 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/output.log -2023-10-23 09:01:10,187 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:10,187 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:10,187 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:10,188 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:10,698 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:10,698 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:10,698 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:10,699 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:11,091 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:11,092 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:11,092 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:11,092 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:11,126 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:11,126 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/output.log -2023-10-23 09:01:11,522 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:11,523 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:11,523 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:11,523 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:12,105 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:12,105 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:12,106 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:12,106 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:12,127 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:12,127 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/output.log -2023-10-23 09:01:12,546 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:12,546 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:12,546 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:12,547 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:13,004 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:13,005 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:13,005 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:13,005 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:13,127 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:13,503 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:13,504 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:13,504 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:13,506 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:13,931 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:13,932 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:13,932 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:13,933 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:14,128 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:14,378 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:14,378 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:14,378 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:14,379 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:01:14,379 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:14,841 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:14,841 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:14,842 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:14,842 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:15,129 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:15,222 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:15,222 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:15,223 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:15,223 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:15,728 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:15,728 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:15,729 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:15,731 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:16,129 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:16,151 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:16,151 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:16,151 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:16,152 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:16,692 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:16,692 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:16,693 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:16,693 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:17,064 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:17,064 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:17,065 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:17,065 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:17,130 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:17,618 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:17,618 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:17,619 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:17,619 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:18,119 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:18,120 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:18,120 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:18,122 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:18,131 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:18,572 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:18,573 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:18,573 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:18,574 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:19,001 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:19,001 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:19,002 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:19,002 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:19,132 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:19,451 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:19,451 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:19,452 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:19,452 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:01:19,452 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:19,878 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 09:01:19,879 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: stop_status -2023-10-23 09:01:19,991 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:20,096 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:20,096 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:20,097 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:20,133 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:20,454 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:20,455 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:20,455 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:20,457 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:20,896 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:20,896 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:20,896 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:20,897 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:21,134 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:21,361 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:21,361 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:21,362 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:21,363 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:21,788 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:21,789 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:21,789 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:21,790 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:22,134 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:22,240 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:22,240 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:22,241 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:22,241 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:22,703 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:22,704 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:22,704 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:22,706 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:23,135 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:23,142 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:23,142 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:23,142 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:23,143 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:23,695 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:23,695 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:23,695 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:23,696 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:24,136 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:24,141 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:24,141 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:24,141 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:24,142 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:24,682 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:24,683 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:24,683 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:24,683 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:01:24,684 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:25,067 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:25,067 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:25,067 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:25,069 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:25,136 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:25,489 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:25,490 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:25,490 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:25,491 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:25,873 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:25,873 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:25,873 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:25,874 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:26,137 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:26,324 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:26,325 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:26,325 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:26,326 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:26,779 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:26,779 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:26,779 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:26,780 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:27,138 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:27,257 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:27,257 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:27,257 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:27,259 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:27,730 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:27,731 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:27,731 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:27,732 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:28,139 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:28,166 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:28,167 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:28,167 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:28,168 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:28,552 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:28,553 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:28,553 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:28,553 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:29,033 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:29,034 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:29,034 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:29,034 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:29,139 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:29,478 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:29,478 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:29,478 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:29,480 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:29,914 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:29,914 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:29,918 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:01:30,125 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:30,125 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:30,140 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:30,140 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/config.yaml -2023-10-23 09:01:30,425 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:30,426 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:30,426 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:30,426 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:30,825 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:30,826 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:30,826 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:30,827 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:31,141 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:31,265 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:31,266 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:31,266 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:31,267 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:31,770 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:31,770 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:31,770 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:31,772 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:32,141 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:32,213 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:32,214 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:32,214 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:32,215 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:32,711 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:32,712 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:32,712 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:32,713 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:33,142 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:33,158 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:33,158 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:33,159 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:33,159 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:33,582 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:33,583 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:33,583 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:33,584 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:34,013 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:34,013 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:34,013 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:34,019 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:34,143 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:34,457 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:34,457 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:34,458 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:34,458 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:34,878 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 09:01:34,879 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: stop_status -2023-10-23 09:01:34,944 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:35,104 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:35,104 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:35,110 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:35,144 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:35,144 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/output.log -2023-10-23 09:01:35,380 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:35,381 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:35,381 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:35,381 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:01:35,382 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:35,803 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:35,804 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:35,804 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:35,805 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:36,144 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:36,268 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:36,269 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:36,269 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:36,271 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:36,633 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:36,634 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:36,634 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:36,635 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:37,094 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:37,095 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:37,095 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:37,096 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:37,145 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:37,585 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:37,586 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:37,586 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:37,587 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:38,027 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:38,027 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:38,027 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:38,028 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:38,146 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:38,576 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:38,577 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:38,577 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:38,579 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:39,051 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:39,052 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:39,052 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:39,053 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:39,146 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:39,497 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:39,497 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:39,498 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:39,499 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:39,951 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:39,951 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:39,952 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:39,952 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:40,147 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:40,372 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:40,373 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:40,373 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:40,373 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:40,816 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:40,817 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:40,817 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:40,818 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:01:40,819 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:41,148 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:41,383 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:41,384 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:41,384 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:41,385 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:41,829 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:41,829 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:41,830 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:41,830 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:42,148 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:42,290 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:42,290 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:42,290 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:42,291 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:42,807 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:42,807 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:42,807 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:42,808 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:43,149 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:43,228 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:43,229 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:43,229 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:43,231 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:43,729 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:43,730 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:43,730 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:43,731 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:44,150 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:44,201 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:44,201 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:44,201 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:44,202 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:44,673 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:44,674 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:44,674 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:44,675 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:45,105 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:45,106 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:45,106 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:45,109 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:45,151 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:45,561 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:45,562 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:45,562 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:45,564 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:45,948 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:45,948 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:45,949 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:45,949 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:01:45,949 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:46,153 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:46,422 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:46,423 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:46,423 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:46,424 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:46,893 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:46,893 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:46,893 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:46,894 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:47,154 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:47,396 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:47,397 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:47,397 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:47,398 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:47,821 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:47,821 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:47,821 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:47,823 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:48,155 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:48,189 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:48,189 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:48,189 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:48,190 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:48,615 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:48,616 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:48,616 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:48,617 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:49,085 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:49,086 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:49,086 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:49,087 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:49,156 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:49,534 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:49,535 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:49,535 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:49,536 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:49,878 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 09:01:49,879 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: stop_status -2023-10-23 09:01:50,048 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:50,156 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:50,168 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:50,168 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:50,170 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:50,412 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:50,412 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:50,412 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:50,413 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:50,805 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:50,805 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:50,805 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:50,806 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:51,157 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:51,282 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:51,282 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:51,283 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:51,283 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:01:51,283 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:51,720 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:51,721 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:51,721 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:51,722 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:52,158 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:52,193 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:52,193 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:52,193 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:52,195 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:52,626 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:52,627 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:52,627 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:52,627 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:53,049 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:53,050 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:53,050 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:53,051 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:53,159 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:53,446 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:53,446 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:53,446 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:53,447 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:53,881 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:53,881 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:53,881 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:53,882 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:54,164 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:54,493 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:54,493 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:54,494 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:54,496 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:54,936 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:54,936 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:54,937 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:54,937 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:55,165 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:55,336 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:55,337 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:55,337 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:55,338 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:55,878 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:55,879 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:55,879 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:55,880 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:56,165 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:56,369 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:56,369 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:56,369 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:56,369 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:01:56,370 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:56,753 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:56,754 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:56,754 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:56,756 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:57,166 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:57,166 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/output.log -2023-10-23 09:01:57,212 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:57,213 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:57,213 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:57,214 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:57,688 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:57,689 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:57,689 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:57,690 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:58,167 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:58,274 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:58,274 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:58,275 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:58,275 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:58,844 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:58,844 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:58,845 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:58,845 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:59,167 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:59,305 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:59,305 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:59,305 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:59,306 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:59,448 DEBUG SystemMonitor:51707 [system_monitor.py:_start():159] Starting system metrics aggregation loop -2023-10-23 09:01:59,450 DEBUG SenderThread:51707 [sender.py:send():375] send: stats -2023-10-23 09:01:59,720 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:59,721 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:59,721 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:59,721 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:00,168 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:00,187 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:00,188 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:00,188 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:00,189 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:00,665 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:00,666 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:00,666 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:00,667 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:01,080 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:01,081 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:01,081 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:01,082 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:01,169 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:01,546 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:01,546 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:01,546 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:01,547 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:02:01,547 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:01,943 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:01,944 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:01,944 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:01,945 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:02,170 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:02,420 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:02,421 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:02,421 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:02,421 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:02,866 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:02,867 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:02,867 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:02,868 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:03,170 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:03,327 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:03,328 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:03,328 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:03,329 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:03,776 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:03,777 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:03,777 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:03,778 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:04,171 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:04,217 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:04,218 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:04,218 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:04,219 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:04,701 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:04,702 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:04,702 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:04,703 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:04,878 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 09:02:04,879 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: stop_status -2023-10-23 09:02:05,083 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:05,170 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:05,170 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:05,171 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:05,172 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:05,556 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:05,556 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:05,556 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:05,557 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:06,036 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:06,036 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:06,036 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:06,037 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:06,172 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:06,509 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:06,510 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:06,510 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:06,511 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:06,970 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:06,970 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:06,971 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:06,972 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:02:06,973 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:07,173 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:07,520 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:07,521 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:07,521 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:07,522 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:07,963 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:07,964 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:07,964 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:07,965 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:08,174 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:08,373 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:08,374 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:08,374 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:08,375 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:08,849 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:08,849 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:08,849 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:08,850 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:09,174 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:09,268 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:09,269 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:09,269 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:09,270 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:09,771 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:09,771 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:09,771 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:09,772 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:10,175 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:10,213 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:10,214 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:10,214 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:10,214 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:10,639 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:10,640 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:10,640 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:10,641 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:11,086 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:11,087 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:11,087 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:11,088 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:11,176 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:11,483 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:11,483 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:11,483 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:11,484 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:11,909 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:11,909 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:11,909 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:11,910 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:12,176 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:12,414 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:12,415 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:12,415 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:12,415 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:02:12,416 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:12,841 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:12,841 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:12,842 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:12,842 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:13,177 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:13,283 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:13,283 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:13,283 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:13,284 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:13,752 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:13,752 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:13,753 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:13,753 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:14,178 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:14,211 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:14,212 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:14,212 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:14,213 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:14,705 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:14,706 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:14,706 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:14,707 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:15,179 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:15,250 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:15,251 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:15,251 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:15,251 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:15,755 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:15,755 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:15,756 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:15,756 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:16,179 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:16,261 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:16,262 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:16,262 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:16,263 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:16,748 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:16,749 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:16,749 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:16,750 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:17,180 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:17,257 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:17,257 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:17,257 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:17,259 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:17,620 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:17,620 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:17,621 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:17,621 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:02:17,621 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:18,093 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:18,093 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:18,094 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:18,094 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:18,181 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:18,528 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:18,529 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:18,529 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:18,530 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:18,974 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:18,975 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:18,975 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:18,976 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:19,181 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:19,182 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/output.log -2023-10-23 09:02:19,491 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:19,492 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:19,492 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:19,493 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:19,878 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 09:02:19,879 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: stop_status -2023-10-23 09:02:20,048 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:20,116 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:20,117 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:20,117 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:20,182 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:20,485 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:20,485 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:20,486 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:20,486 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:20,879 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:20,880 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:20,880 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:20,881 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:21,183 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:21,424 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:21,425 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:21,425 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:21,426 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:21,980 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:21,981 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:21,981 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:21,982 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:22,184 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:22,594 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:22,595 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:22,595 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:22,596 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:23,051 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:23,052 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:23,052 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:23,052 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:02:23,053 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:23,184 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:23,555 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:23,555 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:23,555 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:23,556 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:24,099 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:24,100 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:24,100 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:24,101 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:24,185 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:24,483 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:24,483 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:24,483 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:24,484 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:24,950 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:24,951 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:24,951 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:24,952 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:25,186 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:25,410 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:25,411 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:25,411 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:25,412 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:25,927 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:25,928 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:25,928 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:25,929 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:26,186 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:26,480 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:26,481 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:26,481 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:26,482 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:26,879 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:26,879 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:26,879 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:26,880 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:27,187 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:27,320 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:27,320 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:27,320 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:27,322 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:27,779 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:27,779 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:27,780 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:27,780 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:28,188 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:28,298 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:28,298 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:28,299 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:28,299 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:02:28,299 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:28,793 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:28,794 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:28,794 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:28,794 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:29,177 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:29,178 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:29,178 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:29,179 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:29,188 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:29,451 DEBUG SenderThread:51707 [sender.py:send():375] send: stats -2023-10-23 09:02:29,639 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:29,639 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:29,639 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:29,640 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:30,115 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:30,116 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:30,116 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:30,117 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:30,189 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:30,593 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:30,594 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:30,594 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:30,595 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:31,077 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:31,077 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:31,078 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:31,078 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:31,190 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:31,539 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:31,539 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:31,539 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:31,540 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:32,003 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:32,003 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:32,003 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:32,004 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:32,191 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:32,448 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:32,449 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:32,449 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:32,449 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:32,947 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:32,948 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:32,948 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:32,949 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:33,191 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:33,454 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:33,455 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:33,455 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:33,455 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:02:33,456 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:33,901 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:33,901 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:33,901 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:33,902 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:34,192 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:34,371 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:34,371 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:34,372 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:34,372 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:34,811 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:34,812 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:34,812 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:34,813 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:34,879 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 09:02:34,879 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: stop_status -2023-10-23 09:02:35,193 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:35,204 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:35,205 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:35,205 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:35,206 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:35,635 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:35,635 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:35,635 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:35,636 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:36,088 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:36,089 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:36,089 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:36,090 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:36,193 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:36,467 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:36,468 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:36,468 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:36,469 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:36,840 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:36,841 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:36,841 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:36,842 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:37,194 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:37,306 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:37,306 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:37,307 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:37,307 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:37,643 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:37,643 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:37,644 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:37,646 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:38,195 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:38,217 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:38,218 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:38,218 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:38,219 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:38,714 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:38,715 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:38,715 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:38,715 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:02:38,716 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:39,095 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:39,095 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:39,096 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:39,096 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:39,196 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:39,560 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:39,561 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:39,561 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:39,562 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:40,054 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:40,055 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:40,055 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:40,056 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:40,196 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:40,491 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:40,492 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:40,492 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:40,493 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:41,037 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:41,038 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:41,038 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:41,039 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:41,197 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:41,483 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:41,484 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:41,484 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:41,485 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:42,001 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:42,002 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:42,002 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:42,003 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:42,198 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:42,463 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:42,463 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:42,463 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:42,465 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:42,895 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:42,896 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:42,896 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:42,897 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:43,198 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:43,198 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/output.log -2023-10-23 09:02:43,329 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:43,329 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:43,329 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:43,330 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:43,809 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:43,809 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:43,810 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:43,810 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:02:43,810 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:44,199 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:44,257 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:44,257 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:44,258 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:44,258 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:44,669 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:44,670 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:44,670 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:44,670 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:45,138 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:45,138 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:45,139 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:45,139 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:45,200 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:45,617 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:45,617 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:45,618 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:45,618 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:46,129 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:46,129 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:46,129 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:46,130 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:46,200 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:46,572 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:46,573 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:46,573 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:46,574 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:47,039 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:47,039 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:47,040 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:47,040 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:47,201 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:47,497 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:47,498 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:47,498 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:47,499 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:48,072 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:48,073 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:48,073 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:48,075 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:48,202 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:48,529 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:48,530 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:48,530 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:48,531 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:48,891 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:48,891 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:48,892 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:48,892 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:02:48,892 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:49,202 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:49,346 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:49,346 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:49,346 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:49,347 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:49,787 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:49,787 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:49,787 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:49,788 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:49,882 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 09:02:49,883 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: stop_status -2023-10-23 09:02:50,172 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:50,173 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:50,173 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:50,174 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:50,203 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:50,621 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:50,622 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:50,622 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:50,623 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:51,004 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:51,004 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:51,005 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:51,005 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:51,204 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:51,548 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:51,549 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:51,549 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:51,550 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:51,932 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:51,932 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:51,932 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:51,933 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:52,204 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:52,555 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:52,555 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:52,556 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:52,556 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:52,938 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:52,938 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:52,939 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:52,939 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:53,205 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:53,311 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:53,311 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:53,312 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:53,312 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:53,679 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:53,679 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:53,679 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:53,680 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:54,206 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:54,236 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:54,236 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:54,236 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:54,236 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:02:54,237 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:54,756 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:54,757 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:54,757 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:54,757 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:55,182 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:55,183 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:55,183 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:55,184 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:55,207 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:55,607 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:55,608 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:55,608 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:55,609 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:56,187 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:56,187 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:56,188 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:56,188 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:56,207 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:56,656 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:56,657 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:56,657 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:56,658 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:57,111 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:57,112 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:57,112 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:57,114 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:57,208 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:57,505 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:57,506 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:57,506 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:57,506 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:57,950 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:57,951 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:57,951 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:57,952 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:58,209 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:58,428 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:58,429 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:58,429 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:58,431 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:59,050 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:59,051 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:59,051 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:59,052 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:59,209 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:59,452 DEBUG SenderThread:51707 [sender.py:send():375] send: stats -2023-10-23 09:02:59,452 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:02:59,484 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:59,484 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:59,485 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:59,485 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:59,930 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:59,931 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:59,931 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:59,932 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:00,210 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:00,407 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:00,408 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:00,408 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:00,409 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:00,874 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:00,875 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:00,875 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:00,876 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:01,211 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:01,246 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:01,247 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:01,247 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:01,248 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:01,798 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:01,799 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:01,799 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:01,800 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:02,212 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:02,238 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:02,239 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:02,239 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:02,240 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:02,683 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:02,683 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:02,683 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:02,684 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:03,127 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:03,128 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:03,128 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:03,129 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:03,213 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:03,593 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:03,593 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:03,593 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:03,594 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:04,036 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:04,037 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:04,037 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:04,038 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:04,213 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:04,457 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:04,458 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:04,458 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:04,458 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:03:04,459 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:04,859 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:04,860 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:04,860 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:04,861 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:04,880 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 09:03:04,880 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: stop_status -2023-10-23 09:03:05,214 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:05,357 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:05,358 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:05,358 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:05,359 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:05,793 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:05,794 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:05,794 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:05,795 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:06,215 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:06,238 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:06,239 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:06,239 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:06,240 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:06,691 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:06,692 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:06,692 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:06,693 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:07,153 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:07,153 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:07,154 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:07,155 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:07,215 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:07,216 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/output.log -2023-10-23 09:03:07,629 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:07,629 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:07,629 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:07,630 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:08,020 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:08,021 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:08,021 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:08,022 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:08,216 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:08,446 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:08,447 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:08,447 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:08,449 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:08,947 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:08,948 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:08,948 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:08,949 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:09,217 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:09,423 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:09,424 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:09,424 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:09,425 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:09,808 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:09,809 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:09,809 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:09,809 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:03:09,810 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:10,217 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:10,328 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:10,329 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:10,329 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:10,329 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:10,808 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:10,808 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:10,809 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:10,809 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:11,218 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:11,274 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:11,274 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:11,274 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:11,275 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:11,727 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:11,728 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:11,728 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:11,728 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:12,121 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:12,121 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:12,121 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:12,122 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:12,219 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:12,558 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:12,558 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:12,559 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:12,559 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:13,030 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:13,030 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:13,031 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:13,031 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:13,219 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:13,457 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:13,457 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:13,458 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:13,458 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:13,896 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:13,897 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:13,897 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:13,898 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:14,220 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:14,270 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:14,271 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:14,271 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:14,271 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:14,741 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:14,741 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:14,741 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:14,742 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:15,221 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:15,276 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:15,276 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:15,276 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:15,276 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:03:15,277 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:15,654 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:15,655 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:15,655 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:15,655 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:16,070 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:16,070 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:16,071 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:16,071 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:16,222 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:16,467 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:16,468 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:16,468 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:16,469 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:16,974 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:16,975 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:16,975 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:16,976 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:17,222 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:17,498 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:17,498 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:17,499 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:17,499 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:17,944 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:17,944 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:17,944 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:17,945 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:18,223 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:18,395 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:18,395 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:18,395 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:18,396 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:18,829 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:18,829 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:18,829 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:18,831 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:19,217 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:19,217 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:19,217 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:19,218 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:19,224 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:19,645 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:19,646 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:19,646 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:19,647 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:19,880 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 09:03:19,881 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: stop_status -2023-10-23 09:03:20,190 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:20,190 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:20,190 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:20,191 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:20,224 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:20,703 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:20,703 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:20,704 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:20,704 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:03:20,704 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:21,224 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:21,224 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:21,224 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:21,226 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:21,226 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:21,781 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:21,782 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:21,782 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:21,783 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:22,226 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:22,297 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:22,298 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:22,298 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:22,299 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:22,746 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:22,746 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:22,746 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:22,747 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:23,195 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:23,196 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:23,196 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:23,197 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:23,227 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:23,651 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:23,651 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:23,652 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:23,652 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:24,027 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:24,027 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:24,027 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:24,028 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:24,228 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:24,497 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:24,498 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:24,498 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:24,499 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:24,899 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:24,899 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:24,899 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:24,900 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:25,229 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:25,324 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:25,324 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:25,324 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:25,325 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:25,914 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:25,914 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:25,915 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:25,915 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:03:25,916 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:26,229 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:26,394 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:26,394 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:26,395 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:26,395 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:26,839 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:26,839 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:26,839 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:26,840 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:27,230 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:27,269 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:27,270 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:27,270 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:27,271 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:27,672 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:27,673 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:27,673 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:27,674 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:28,153 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:28,154 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:28,154 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:28,155 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:28,231 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:28,584 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:28,585 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:28,585 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:28,586 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:29,064 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:29,064 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:29,064 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:29,066 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:29,231 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:29,231 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/output.log -2023-10-23 09:03:29,452 DEBUG SenderThread:51707 [sender.py:send():375] send: stats -2023-10-23 09:03:29,506 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:29,507 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:29,507 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:29,508 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:29,896 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:29,896 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:29,897 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:29,897 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:30,232 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:30,320 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:30,321 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:30,321 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:30,322 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:30,738 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:30,738 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:30,739 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:30,739 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:31,130 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:31,130 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:31,131 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:31,131 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:03:31,131 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:31,232 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:31,517 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:31,517 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:31,518 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:31,518 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:31,947 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:31,947 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:31,948 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:31,948 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:32,233 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:32,529 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:32,530 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:32,530 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:32,531 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:33,006 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:33,007 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:33,007 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:33,007 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:33,234 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:33,458 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:33,459 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:33,459 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:33,460 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:33,894 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:33,894 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:33,895 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:33,895 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:34,234 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:34,341 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:34,341 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:34,341 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:34,342 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:34,800 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:34,801 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:34,801 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:34,802 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:34,888 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 09:03:34,888 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: stop_status -2023-10-23 09:03:35,235 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:35,246 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:35,246 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:35,246 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:35,247 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:35,687 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:35,688 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:35,688 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:35,688 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:36,106 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:36,107 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:36,107 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:36,112 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:36,236 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:36,633 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:36,634 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:36,634 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:36,634 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:03:36,635 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:37,071 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:37,072 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:37,072 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:37,073 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:37,237 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:37,514 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:37,514 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:37,515 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:37,518 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:37,912 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:37,913 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:37,913 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:37,914 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:38,237 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:38,343 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:38,344 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:38,344 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:38,345 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:38,866 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:38,866 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:38,867 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:38,867 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:39,238 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:39,296 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:39,296 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:39,296 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:39,298 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:39,820 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:39,821 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:39,821 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:39,822 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:40,239 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:40,260 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:40,261 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:40,261 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:40,262 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:40,682 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:40,682 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:40,682 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:40,683 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:41,130 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:41,131 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:41,131 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:41,132 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:41,239 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:41,600 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:41,600 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:41,601 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:41,601 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:42,162 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:42,162 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:42,163 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:42,163 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:03:42,163 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:42,240 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:42,559 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:42,560 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:42,560 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:42,561 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:43,148 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:43,148 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:43,148 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:43,149 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:43,241 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:43,490 DEBUG SenderThread:51707 [sender.py:send():375] send: exit -2023-10-23 09:03:43,490 INFO SenderThread:51707 [sender.py:send_exit():598] handling exit code: 255 -2023-10-23 09:03:43,491 INFO SenderThread:51707 [sender.py:send_exit():600] handling runtime: 164 -2023-10-23 09:03:43,492 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:43,492 INFO SenderThread:51707 [sender.py:send_exit():606] send defer -2023-10-23 09:03:43,492 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:03:43,492 INFO HandlerThread:51707 [handler.py:handle_request_defer():170] handle defer: 0 -2023-10-23 09:03:43,492 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: defer -2023-10-23 09:03:43,492 INFO SenderThread:51707 [sender.py:send_request_defer():622] handle sender defer: 0 -2023-10-23 09:03:43,492 INFO SenderThread:51707 [sender.py:transition_state():626] send defer: 1 -2023-10-23 09:03:43,492 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:03:43,492 INFO HandlerThread:51707 [handler.py:handle_request_defer():170] handle defer: 1 -2023-10-23 09:03:43,492 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: defer -2023-10-23 09:03:43,492 INFO SenderThread:51707 [sender.py:send_request_defer():622] handle sender defer: 1 -2023-10-23 09:03:43,492 INFO SenderThread:51707 [sender.py:transition_state():626] send defer: 2 -2023-10-23 09:03:43,493 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:03:43,493 INFO HandlerThread:51707 [handler.py:handle_request_defer():170] handle defer: 2 -2023-10-23 09:03:43,493 INFO HandlerThread:51707 [system_monitor.py:finish():190] Stopping system monitor -2023-10-23 09:03:43,493 DEBUG SystemMonitor:51707 [system_monitor.py:_start():166] Finished system metrics aggregation loop -2023-10-23 09:03:43,493 INFO HandlerThread:51707 [interfaces.py:finish():202] Joined cpu monitor -2023-10-23 09:03:43,493 DEBUG SystemMonitor:51707 [system_monitor.py:_start():170] Publishing last batch of metrics -2023-10-23 09:03:43,493 INFO HandlerThread:51707 [interfaces.py:finish():202] Joined disk monitor -2023-10-23 09:03:43,598 INFO HandlerThread:51707 [interfaces.py:finish():202] Joined gpu monitor -2023-10-23 09:03:43,598 INFO HandlerThread:51707 [interfaces.py:finish():202] Joined memory monitor -2023-10-23 09:03:43,599 INFO HandlerThread:51707 [interfaces.py:finish():202] Joined network monitor -2023-10-23 09:03:43,599 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: defer -2023-10-23 09:03:43,599 INFO SenderThread:51707 [sender.py:send_request_defer():622] handle sender defer: 2 -2023-10-23 09:03:43,599 INFO SenderThread:51707 [sender.py:transition_state():626] send defer: 3 -2023-10-23 09:03:43,599 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:03:43,599 INFO HandlerThread:51707 [handler.py:handle_request_defer():170] handle defer: 3 -2023-10-23 09:03:43,599 DEBUG SenderThread:51707 [sender.py:send():375] send: stats -2023-10-23 09:03:43,600 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: defer -2023-10-23 09:03:43,600 INFO SenderThread:51707 [sender.py:send_request_defer():622] handle sender defer: 3 -2023-10-23 09:03:43,600 INFO SenderThread:51707 [sender.py:transition_state():626] send defer: 4 -2023-10-23 09:03:43,600 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:03:43,600 INFO HandlerThread:51707 [handler.py:handle_request_defer():170] handle defer: 4 -2023-10-23 09:03:43,600 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: defer -2023-10-23 09:03:43,600 INFO SenderThread:51707 [sender.py:send_request_defer():622] handle sender defer: 4 -2023-10-23 09:03:43,600 INFO SenderThread:51707 [sender.py:transition_state():626] send defer: 5 -2023-10-23 09:03:43,600 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:03:43,600 INFO HandlerThread:51707 [handler.py:handle_request_defer():170] handle defer: 5 -2023-10-23 09:03:43,600 DEBUG SenderThread:51707 [sender.py:send():375] send: summary -2023-10-23 09:03:43,601 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:43,601 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: defer -2023-10-23 09:03:43,601 INFO SenderThread:51707 [sender.py:send_request_defer():622] handle sender defer: 5 -2023-10-23 09:03:43,601 INFO SenderThread:51707 [sender.py:transition_state():626] send defer: 6 -2023-10-23 09:03:43,601 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:03:43,601 INFO HandlerThread:51707 [handler.py:handle_request_defer():170] handle defer: 6 -2023-10-23 09:03:43,601 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: defer -2023-10-23 09:03:43,601 INFO SenderThread:51707 [sender.py:send_request_defer():622] handle sender defer: 6 -2023-10-23 09:03:43,602 INFO SenderThread:51707 [sender.py:transition_state():626] send defer: 7 -2023-10-23 09:03:43,602 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:03:43,602 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:03:43,602 INFO HandlerThread:51707 [handler.py:handle_request_defer():170] handle defer: 7 -2023-10-23 09:03:43,602 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: defer -2023-10-23 09:03:43,602 INFO SenderThread:51707 [sender.py:send_request_defer():622] handle sender defer: 7 -2023-10-23 09:03:44,241 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:44,490 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: poll_exit -2023-10-23 09:03:46,243 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/output.log -2023-10-23 09:03:47,200 INFO SenderThread:51707 [sender.py:transition_state():626] send defer: 8 -2023-10-23 09:03:47,200 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: poll_exit -2023-10-23 09:03:47,200 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:03:47,200 INFO HandlerThread:51707 [handler.py:handle_request_defer():170] handle defer: 8 -2023-10-23 09:03:47,200 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: defer -2023-10-23 09:03:47,200 INFO SenderThread:51707 [sender.py:send_request_defer():622] handle sender defer: 8 -2023-10-23 09:03:47,210 INFO SenderThread:51707 [sender.py:transition_state():626] send defer: 9 -2023-10-23 09:03:47,210 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:03:47,210 DEBUG SenderThread:51707 [sender.py:send():375] send: artifact -2023-10-23 09:03:47,210 INFO HandlerThread:51707 [handler.py:handle_request_defer():170] handle defer: 9 -2023-10-23 09:03:47,243 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/output.log -2023-10-23 09:03:47,294 WARNING StreamThr :51707 [internal.py:is_dead():414] Internal process exiting, parent pid 51105 disappeared -2023-10-23 09:03:47,295 ERROR StreamThr :51707 [internal.py:wandb_internal():152] Internal process shutdown. -2023-10-23 09:03:48,212 INFO HandlerThread:51707 [handler.py:finish():845] shutting down handler -2023-10-23 09:03:48,212 INFO WriterThread:51707 [datastore.py:close():298] close: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/run-141dzr9c.wandb -2023-10-23 09:03:48,216 INFO SenderThread:51707 [sender.py:send_artifact():1474] sent artifact job-https___github.com_junchen14_MiniGPT-4_finetune_train.py - {'id': 'QXJ0aWZhY3Q6NjEzMjczODU5', 'digest': 'e1357229d7190882bf8b2dba86a37a56', 'state': 'COMMITTED', 'aliases': [{'artifactCollectionName': 'job-https___github.com_junchen14_MiniGPT-4_finetune_train.py', 'alias': 'latest'}, {'artifactCollectionName': 'job-https___github.com_junchen14_MiniGPT-4_finetune_train.py', 'alias': 'v0'}], 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjEwOTExOTI4MQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6NjEzMjczODU5', 'versionIndex': 0}}, 'version': 'v0'} -2023-10-23 09:03:48,216 INFO SenderThread:51707 [sender.py:finish():1550] shutting down sender -2023-10-23 09:03:48,216 INFO SenderThread:51707 [dir_watcher.py:finish():365] shutting down directory watcher -2023-10-23 09:03:48,244 INFO SenderThread:51707 [dir_watcher.py:finish():395] scan: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files -2023-10-23 09:03:48,245 INFO SenderThread:51707 [dir_watcher.py:finish():409] scan save: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/conda-environment.yaml conda-environment.yaml -2023-10-23 09:03:48,245 INFO SenderThread:51707 [dir_watcher.py:finish():409] scan save: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/config.yaml config.yaml -2023-10-23 09:03:48,247 INFO SenderThread:51707 [dir_watcher.py:finish():409] scan save: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json wandb-summary.json -2023-10-23 09:03:48,250 INFO SenderThread:51707 [dir_watcher.py:finish():409] scan save: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/output.log output.log -2023-10-23 09:03:48,252 INFO SenderThread:51707 [dir_watcher.py:finish():409] scan save: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/requirements.txt requirements.txt -2023-10-23 09:03:48,252 INFO SenderThread:51707 [dir_watcher.py:finish():409] scan save: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-metadata.json wandb-metadata.json -2023-10-23 09:03:48,253 INFO SenderThread:51707 [file_pusher.py:finish():168] shutting down file pusher -2023-10-23 09:03:48,255 INFO SenderThread:51707 [file_pusher.py:join():173] waiting for file pusher -2023-10-23 09:03:48,833 INFO wandb-upload_0:51707 [upload_job.py:push():137] Uploaded file /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/conda-environment.yaml -2023-10-23 09:03:49,124 INFO wandb-upload_1:51707 [upload_job.py:push():137] Uploaded file /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/config.yaml -2023-10-23 09:03:49,247 INFO wandb-upload_2:51707 [upload_job.py:push():137] Uploaded file /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:49,256 INFO wandb-upload_4:51707 [upload_job.py:push():137] Uploaded file /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/requirements.txt -2023-10-23 09:03:49,326 INFO wandb-upload_3:51707 [upload_job.py:push():137] Uploaded file /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/output.log diff --git a/wandb/run-20231023_090058-141dzr9c/logs/debug.log b/wandb/run-20231023_090058-141dzr9c/logs/debug.log deleted file mode 100644 index d453118..0000000 --- a/wandb/run-20231023_090058-141dzr9c/logs/debug.log +++ /dev/null @@ -1,26 +0,0 @@ -2023-10-23 09:00:58,119 INFO MainThread:51105 [wandb_setup.py:_flush():76] Configure stats pid to 51105 -2023-10-23 09:00:58,119 INFO MainThread:51105 [wandb_setup.py:_flush():76] Loading settings from /home/chenj0g/.config/wandb/settings -2023-10-23 09:00:58,119 INFO MainThread:51105 [wandb_setup.py:_flush():76] Loading settings from /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/settings -2023-10-23 09:00:58,119 INFO MainThread:51105 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} -2023-10-23 09:00:58,119 INFO MainThread:51105 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'train.py', 'program': '/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/train.py'} -2023-10-23 09:00:58,119 INFO MainThread:51105 [wandb_init.py:_log_setup():507] Logging user logs to /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/logs/debug.log -2023-10-23 09:00:58,119 INFO MainThread:51105 [wandb_init.py:_log_setup():508] Logging internal logs to /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/logs/debug-internal.log -2023-10-23 09:00:58,119 INFO MainThread:51105 [wandb_init.py:init():547] calling init triggers -2023-10-23 09:00:58,119 INFO MainThread:51105 [wandb_init.py:init():554] wandb.init called with sweep_config: {} -config: {} -2023-10-23 09:00:58,119 INFO MainThread:51105 [wandb_init.py:init():595] starting backend -2023-10-23 09:00:58,119 INFO MainThread:51105 [wandb_init.py:init():599] setting up manager -2023-10-23 09:00:58,121 INFO MainThread:51105 [backend.py:_multiprocessing_setup():106] multiprocessing start_methods=fork,spawn,forkserver, using: spawn -2023-10-23 09:00:58,125 INFO MainThread:51105 [wandb_init.py:init():605] backend started and connected -2023-10-23 09:00:58,127 INFO MainThread:51105 [wandb_init.py:init():695] updated telemetry -2023-10-23 09:00:58,534 INFO MainThread:51105 [wandb_init.py:init():732] communicating run to backend with 60.0 second timeout -2023-10-23 09:00:59,118 INFO MainThread:51105 [wandb_run.py:_on_init():2176] communicating current version -2023-10-23 09:00:59,434 INFO MainThread:51105 [wandb_run.py:_on_init():2185] got version response upgrade_message: "wandb version 0.15.12 is available! To upgrade, please run:\n $ pip install wandb --upgrade" - -2023-10-23 09:00:59,434 INFO MainThread:51105 [wandb_init.py:init():782] starting run threads in backend -2023-10-23 09:01:04,878 INFO MainThread:51105 [wandb_run.py:_console_start():2157] atexit reg -2023-10-23 09:01:04,878 INFO MainThread:51105 [wandb_run.py:_redirect():2012] redirect: SettingsConsole.WRAP_RAW -2023-10-23 09:01:04,878 INFO MainThread:51105 [wandb_run.py:_redirect():2077] Wrapping output streams. -2023-10-23 09:01:04,878 INFO MainThread:51105 [wandb_run.py:_redirect():2102] Redirects installed. -2023-10-23 09:01:04,879 INFO MainThread:51105 [wandb_init.py:init():824] run started, returning control to user process -2023-10-23 09:01:04,879 INFO MainThread:51105 [wandb_watch.py:watch():51] Watching diff --git a/wandb/run-20231023_090058-141dzr9c/run-141dzr9c.wandb b/wandb/run-20231023_090058-141dzr9c/run-141dzr9c.wandb deleted file mode 100644 index 4459c867ec00052a7644f5169279199ceaa643fe..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 99174 zcmd44349b)_Wo^k5{05sB9%v;C5Nr&?<~ zwN80?muMo|_sqs@pV(pFE{<7Nm(Xx4vhr_tUniSw9*|7b4@#ce+udWDZayv6ef{!v%h#+LT3z3__i!t8);arZ(DqrP>avnSw5;xH#>3fg zG_}i`k#IB-j>XnYgyYe0WS`JZ)>P|+@}2{ln+G@b>e;h@ZS#O3y~BMP2KB5PQr~Al zO?@=c^AKd*|F9!_B&zz=*4H!-sjum_Q?gIIrcb7K?{qxUJ64nJ+b=Uei04 zh(P+91p}3P8=62Zukt+V~-)nk3^EP;}2a0N!IFOtyJNwF3WlQe;95?mNo1C)@_|G z6YN!AzVxxlB^BXVBo^-xjr54cDN|NiHpCsiKNT;14Q+1yyo z64v)GU-e)8)Q;!e^d7%@t-8rpx3#-WvVZrS;Wt_pS6lAio!%`HiM9W3EoxIub5&z? zb8SOYWqm_)Wz*oAKDGU7Yx?e1+4gBu<=^J4}Q;KwS#L0^2yHO02rU$zV7ksz}Tg(CJ%AK z=xJ8P(JMiCa6@%pgo(zY>1btRO`nFwzD zyw87-uxs5`T{@Pb<=KaW`bU;@~9*^SHI^HZ6GE@{6Cq5uI>C?4*-c{gD&Phg>KL*D)lV`OK3@7+|djJ|3Sp zj>oNA*SkQ%S&PP$RUAI3q3@7^HQ|8+s|QtA@h>$4!@aTKwtvk1huq%>R-aMR*c6V0 zn`#F3tEz5pt`DDDHMp^%Ebbb-)T7-(BxeIOXxld`AH7TPTh zw2Oy7R7|wN*Y@+-bB77-RtDNBOCAn}_Us=*+ucAr?$f!!&>lAUoQ;L{uLjx)t@DbBHu%~eFnC~Jq210vJN>m6 zgQ4B;-2SaXyS;&S%#>Gxpp6%O-@bdnHO~v}4hGs~!(R)A_U%Kr+gxaOG|;xZ@J2Dw z24CBe&t-2H+MNuvOXs~64DH|dS+S?k?rfl4`t&=ii?%rXc8BM8Y!cdC477`{em5A} zFR#1S&I)!l(2kq3sF-MjukC4rmhLUIyBTQ5{Pun@v?KTGU!}F}VW3_9(?>zjCW^jq zU%2X}kwP0b(9U{$NiekGv0IK2+MXub&p$0D+Td$D<+$(d{5E2sZGGUgU}$$+ddJ>E z8#U05`{Rq%MO&PGd+IJ9KdH5i8EB`>{VEvRzc*d`j?l&pw98-prkH4hukHSKefpZv zCJeM=p7}Ny+8^$pVOMIB2HG(X{TKvovgrHvr)MteBD5(3?XoL>4u-buo0*qrZPNzY z<@fzkOtit*_Ri-%_)KUsCfZTI1w%V-??rzKZPq~BGJnPDqAkw8y{~JpTZFckfp+=C zKZBv&tzyhHq21j;JN>?4YX+#W7Fu#4+Td&Z#yaskg?0}E?euBquNkPqddy~LmkI5j z2HJ5`M+HHfD*C>?;H8UFLi;xZ?VPcrgQ49$Ho~q;?PZ`{de=q8L>qixGEUhtG*s_xX|urpq=x<_#kN0Mc=nw8jiQSUjH=E zPI>gYU}&=kjCxmS_czckf98f_q7A;bw_Nn#G_CCc2HIugZVHCBd#^W}h4w%LZOaEY zuP)l+?AwPvpSY#a{>wl+=FUmM(C+^8<4cA1AOr34yKXHe+Td$jR=w|!LVK`*w)L0E z!O&Jbx%NMV_7DT@^arK}L7OT1zWvu#bE<{*-v-(-m){u-?FEl>p*_q%yWp$4gQ5NGj`~4Dd$@tNW!gQfi?%rX_Sl_XR^+ zb9ndv2<;ID+Sa)b6ccUmwf#Ljbe`7sNCWM-u``3AJ#5d-Zxq_2475|;ek2ImY|;1a zkIxT(U1*Os&`!AXv0!Mg7=G(dLVJvXcG>+;6ccUmwH@{9lJ$i4SOe|2UuFeEyTN4} z94xfQ8E9uc^z`bYEz-Wdqsfkwk2lcHy!e@5Xn%d}&y$7r1OsjB)z1|ZZSb`{xQ8CTU0=cNW5AuTU_nvg24CH+_FM5!!R>3noj&QEU~mV0 z`tluuTVueTvh2M;aEnixI&=507YS}Z1MY;U-wy_N_nAkf1-HKecfso)6cujp)&1KZ zYyBa(0}Qw=<30)o_uggwUlH6|1MaN(OI90harW-3Z`Xb&xOE2H`PY3K4DR{&T>E#y zJ;i`K^M=og3OD%b?)B-c0l8vf_t6;cigX678P#r)!qKvKSv*kd@YuNcgizY z1;abK{*=~hdEPL~0K9a?HLDG{IJ@`Ze*a!efQOram(Lj=4DdCtG;Jxs=bM1%yn0OxBhfXQ2_^E;Q@30_M!l{nt;bWIXM{M+ip92q5zLJ0nZwC`)UI& z&K`dA%d)ow_(Bu#{CQJ@0d5*SVX^>UWCCtobZ1ck2VddsHXRWc;EPSb<8Gf83~=LC zm!<^x5)<(96*B?>E}8w~K|iM>x0;LA+FQ$D=EsDOj7@X5pW z{aJv=n1EX!doUQ_IVYU6o&aBN0vXO9&XaPSpwIq8VI1bD0oc-Es&1Oxn9zn4k{_$m|d{9#WA0$hBGRe0n<;|2I? z6Y!kZX9okkT{xwN0iI|Ao;mZqV1PfL zI(4c5Pci{dc=-LI0uH{y&(7R+fB@fO0-iqdgJ6KW-o2z=fNwPcFMIgoK!A%+y(+n* zW{Ch#HUTfaSw2hyc$p0gt)j_tgemoIU*ezCZU7;Cmc^ z-&+w3aD0c;4i(^gO~9@1|5;SP!B_as+O9hb@O>uWaZ`tN3DhOM{_tu$mcHKvyx{#2 zT>`X8W5s7v1F8zoJc=q7$ks0p}r@)f}VUtM>i9ZNrE0-pZSRe=B(pOy92iDxDS_;C~Pln1T} z2KcVqHnn5vCrrR&W{xi^;NUBK?R}p`1$dSTc*6PD1p|ERsUMBi9)8jUy!_r9RvU0} z_V9oa>AnK|lmqb3HwFW|?^BCo0{pZIcv;KMMFkvug{yuXf1d!)HUW>DJuw*IAN#%h zvH(A00$%XTt$_f0(UpQ<>#lt9YP;3)tOZC;@)X1U&!Q+lvY~_zEwd zaQIBE@EjBHobh)A1Dt*1*zN*6*96@9>Rqc1xHx-w?shZQ5#Z-dz~gSXI~d@LGwVMr zz%MueKRctSfP=5_I!B&;j{wgz0k@35HyGfHvN!G{z%QDB$1Hjv5a8mowCbOE@qPh* z$pqXwV`ebGS0A(6ashtX1ib8?hl>h0_zE}waplPZJl_P|^2Z~=0N;AR)O!T@6%+7+ zX^*cq;NtAzsYkr9o&djU0-ir?RxrSS9k%Bo0{of@c;@*}6%}ys6~5z=iTeof>khy_ zJ{=73w6zv465uyXz;m8@E)d}2^R#w5efl5)e$xcJ;F7t)0MAWqRVTo2nSf_q_Ciqs z2Vda}_xk!~0bXDNUi#|1V1WO#^tc@a_-zyLm@8jiZNSCZ!RiFzY^g0Ou!2kzZD2@@p)P^ zKK?N#z>7@4^Y48-7~u0J^_nff@0);U-oLP@fP=4a^IbdG)WgLl;5jYt1p{0%XZ8UC z{DBF0!i@J<8*p*<@VmGCyg)1bp$T~U?~8*0{^PL47Ygu4CgABmeOOe$!B_b2A0Pac z0Do)(p7rF%!2qAsI`Sz2USa}X`p2h%02iO9bqL<9<#E@E0cFF}HkQRKUSkxZCa@+sD$ECg7Q${tyiCODC?kSb)DW0WZIP zc_6^W=V_&vZEauj_}T=#?3-VL0X{g?@PGh+V*+0A?QcZ|9DIdGz8?RlOnR9Kc>e4a z!2rKM`ZaqGe`^As|42*MAl<{l6)w&m-eB|8RRa8-33$$>!@CA*mo7eTBRiJ<-UQq- zVMI{@2VddWHfyj~_y-g4m@h^K1N_(qO}ojYe>4G4m_9lX;NtVNw%_E4kF>%+nSjTg ze^D^NHzgi8O@Mzk0k@32q^N*{ukb$xKRsN4mz#jc%)c}k;OYOZ8z8{Hn1GkIT)x_X zi?fH9pZ!2qfPXasFMr{RV1V~nW2_xo{$>JR{NmW60uH{ypT4)lVFLWS3Apw0tAhcq zeP+(RTHzHY;OQ@o4+1z|e4f@*o4#Knz<-#4XN|uu7~pM2-EzAC|7ilAa@`F@1sr^Z z_d5UBUj?`&WZWN~zv#wbfL}TN^Xmn8Sjf0PJms>RR~v9~_VDdH{Kf7M4>tkNd}m@X zz;pI!`G)|XZvvkA`YlBT9DIdaHon7lS|d!ri?5j+3~=kdKfWQrBTc}qZ`>XTaPfIs z12)^FyG;556Y#hRcLW1`@MR0%7T{4P;FgJZ6%}ys6>fZ@X|Vvent;cAH7yw6!>g|T zuK!B@EF<}>Yz z$HgY#nKwQV4Dgfn%C-=xw!5mzrWP z`|Z16Vt><|xaq$<@3K(cj)h{c_@QgXrf#u&*H#C3g*}>D#kok@ok4*AE%gyQZ;n!wvD+G;?$9815%4j4KsjQWJm4lJJw!WUf+n)w`t@jnLwCs4G#iig--Q^+Vyw!fY z`r?sS?p`r0)N???pqie22GrEoMf&%wuCL|=>3yr?J!|SuYwB5ZMorIwwY_@|KBIX+ zLw!7)?b(D>JqB0zIiYvdJbu>9oVEt^f_g4Lv4L?c<>p$;?Z7}M>W+n z9#h?@SHkzI{C7irjlBrGqP(_#@Q~)F$^q4<)$l&^Q-{;XS=aG#=YQmNK z^N+PP_54WtTif;R^2*Agxu-RS^$$CBU;Srx-#NgtU;6A`)h*cK*ig5#%GWM0vlnK# zWkt%jEH53{(8PA>77fQEiEJ#AOeUhKXe1rYly6gBUe#PXsHUm8deGqE)&|j3HWNui zGO27hos2|d{6Lqg#v%3k89gByPQ?pEI3al_x|_s;oSrva&MLQx?X*)HGN3 zs*LnXBzwg&m6Zb<`Ij)CL?TiC4J5rPb3iI9B~`D=SQuIO$-dS07ww;|tQ=H*dgY*+ zLA@&DaR9IS>&TCJbMZg_1(K^nbsH4`Y5DC!fCPE{`trHAE*-%GuL*TK&ufJu;cPUL zjYRn~5>F*knPmC4<_dLdU!im)9L*$S@odVuMzL@tnN38~iA*Y*jimX3f;EbE3w5!o z#~*$BFhm&_>UOp%$`1;aTmIEV+CZm~b zHXcu>Q`tf}V#pEd*}CsJQptE9l|;jdY&4sR#xrUDilb>J)Be_HR5Lk47_zNF>QWdaq9`9F4@2 z>9kWRxXyGsolIon(Rh?SZzv^+BxRR@Y(I$^cy6C~ zI2qxnk2;l-3}bE4bcVf^Nyk%3V|g-2(&g=kQ+rA&H~A_hm58Ua7=JV#XQ?w;Kc!^j zkqFM+EeeN1ES*dzqfs`YxjtD$>H6W1cb{|tqD%;NYq7kIHpcOSnXq?Q=tLy$ElWb^ z6P@gzR5+H5Br}nCGy*2HTfR>I1PVnX2va^Tb=eux%gsJ|k*CF8Bk4>!mQG_cJ{u?z z&cw6Hglhw(!nmY#ERjh@QwWnz8KTJ3u6_8MC;UaCO!N^&9u5y0Phlx3Ry+~+HkM>K z#`$d@+{U?_4o9+yR5le!aKJ}1>7*fwob5U%EuL_omS~dy5+RAKG0j1wGstHTB{_Ff zNx}f9xunCX2)++InM@{@AV?@&BDvdj`)#zt{nE=VE740Ll1(L}>0~OFAtd%TmsB{) zF+to?{+GglWy5GDjC114Nt`xL%Cr`_+x6D_<84cNxz%4U@l*!mXA`jt5}1RZEORGN5%J%n>G^dgVD{!VWl|EPqSyb@tDY@u`lb4ey5$(WDN#W0B2_)+H~ zWx`ojHyKN%SY>$_Ll`;T4K^*??RhQIZT^dde@kSOXrDkK8IO7!FLn`4#Un2Fl4S!# z*hSe0!bGykm?4Us?uNU(bms38Ws0vT>0~yA^(T2$D#H?Zi;@XP(I1L)#JAZn(LpMb zj3nZTXf&2I_mRAAx4jaNZLKA`-G7NV>NwIkm{Y`2$*7OLU>4~JhmP|s;<9pr?VP~F z#PAN8sJV~icQ-ol+wZrKUZ(o$g+q};fKw=)?5oqn`Ik0_YtySA~B*LjsWGR zg-aBZ*WLKH=6%a0${oI}ZA?xwqZ zUvr3rxyxUeXfmVonFuwSND;nx`j-r5!L}uqbr^3fhGtUfOeB&a>IYTE5JrA?v!8ms zI7Evy&3BQKNgM{x$glv3ICkfigdhv&c?{p{@Vm&8PH}3Y+9)SPI%|j`zq|S5vXZwI zN8Rm;qY8sdh1;xhHmWQx5%V;c3}%ree91bM5)a1|_@pvQg(Qk0iJa~h^L{D0LXu2( zNm3ZmC3(0`BUF=#$74x9QII*2iaK=?598lbaeQ5xP&Z{Rk34S0ulqmxktCVnlElzS zJdRQkBF!S9G9&m&Lo0H(TVFnE<{Da``&{c&XeqHYp=Fu_gwVy# z61`&o3>J}%N3&_?`Xq@II683t)BrHuLP_HCwtv}uWK~K^x!*@A3M`3LI6M*wwrL{j ztrQFbyNshZ^FWNHPzp|os54F?f8%xT3xf~9@MBUfpiE)pZ7WYbYxnIX%!B^Iq|&KmHW|rA38ZvFdus-(ND&UWY6@s3 zi+_tz2*C2$WyW@n%iC`IdS(rKk!HFUsnA?hMvG*UgbNW$YCe%=CK*l;+NUYCIQDZo z94FS`Y)?}@;-ohgN#3^mXM^rpB)vT3uNTZe$@#&@3Us}~9~?3bi-kqo?yW^!VevvSb_zQ#OQZ`lGLEsq8`U<0BmtrKxUdn`V2vO2EqISw0RI&;+ zBe%Q#bqf~R6|qNsEe6{sZl{ctB3esQrS)_$X|iP&)O9Lm!%@yuTnm4)7s$j5MUmUx z;qj5XE|Fdy^VbX6BGoKHLUs{HiI0!LFqDYlLL7R@k}t+Ggi)$0+a;tzVdQvsJfh-n z_ehw>{e?lf84`8!X#ya$>}@d^MvAb60;?$u%##QQBee$-6<3m$kni1T{o#FImhXMS zcab96&eYckp;DQcPx6`}0l>phX?1LCWJ&O4b~S}yXY!hm@7?*-e%H22lv%!_aPlTG zUwjPK5GCvJw3jUCIlP?qf+XypRFWiw6OvQS@GuE^-d(O9a9lSn(Ubm5#2&<85>#}f z_$!~FJ3|71Wm4mIE|EgL2p)>!C&dy46$SQ^kmucXx7$aL)j|D~??J6>oOPy*l!+7b zdM6{uLdl9#(78g{1fBzf9b5|!7e2GgHWeSpHU*9v82 z6LB&GrzFIBvRcw;;$RAwhEC*ldz{tfA0NtAX8YO-zJTC}C$N{ucru<_2uU&uFXKtK zDC!~+H(>u1QKN<^^10#LzTGJ+QJ(P?g=8kF>IKCpRk^)WqHHvbIV7p#IYlAZQ!Ap`KarhNNy#(cL!dx~$ zg1uyjB99w6q3OwK66HCUD23qy&V)@tmd=h;f!JFXOoG^-kjbf)1bq@ro-~4z8KxFyB88$Pm!&D96o3fdB?{p(Rb?VE zLM+@Q$ZD<6%f3pX_KHuSn#(yt0mwW0M=6wjsIa<@p;VYUr7~y^01{Ut#7*iLN-v!_ zp`Vm8-=&m7FGtxZMOld$jyzOw1fOiB^hRSaArAcmWaYpM)fI935&K!#E4juI)}2&r}eN) z3$MD=QfMoLDa09sVnoc;p}e(1o6P1A#hq45;7bBd{6vCB5@N&&NgYGI`i9YlK^zVQz~%5dLstx%!i589A{@V?lfEiU)!lr{Fa+DB$2n>bKu0O?+-_kH$&adF`KQ0wgOJ7 z+em?C$V0pZQIE`~B)A2khmGSGe}LU+2qItmw~PPVb-qmHEnibntyAGTCx-|JK5?8{ zA>>SS?oGF!;6SE?g}LA+N!kowo06~HtE6SKb0x|GUr|W`!HqO%!Nm3bdDI61A20{&E2g%v~ z%*`jazY5ACpb`ka)=N?g~@3s@i3hLNNIG6Q|xq;9FUfy{pU9Cj<=WR zeP5mM>oI&f4{FFj=JA#Vd%y)M&vEF4syB@oVlQ$>I?skzNz2pjzs5HkHE07Z_TNBJ zvLE~k$sy4O&dyI3vO|sqXHg?T(VPC1s!nV|!ayU)PRrXKuzb6-e$xi}z_o!4w~t0t z0pMJY5#oA_f<@q!Sq!IIk`TtIM^TTC!Mihiy=ghy1GoS8i#JG;4_%TJ9)%3) zM@$*>4&dk!Vh_kH~3#I!EDfSRldJB@GB1jII zaawGKiXgEyc{?Q*BZ|z(%^o)O(Eog|#re#CaVWu3KA_`4q0^=zdIYqvcOV0ZNb2%! z8H!?L$|=?2(zKO~b1)-Ed-&x?-PA{m^SNtr3Mt8i_ZUf<^FwSmZ!@7ErtTimx5H$! zq(xX5*)45X0+m9Cl97}B&-fD$?=P)<;jb0i-gdJ&*Wu+j_0$UXpkgYZ9U&8Kk1RDO zd_85jsJPw*-Yp|Pd&GK?8K-I|f9b!IIl_pmc@zyz@76cG9w_4!L*vfyI!V(bMKzt& zpAG>XMne+$*&}a#a`9dYnZ9zlw?dnxMw(5lGYd_EZoaotXoXRXVjUbxK@#?`cGkj~<_xaEnCw#wALjR?vr(ZifaWAl_QZhE;!2@!m9*B%OSnKIDq@8OTbEq$49& zd(89S41P$WEb|ow?kz1pyd=eFC>NfhM6n2tN^wzLqS!Y`h|-A+@JpuE%F!Ns=5EvM z0>Za0Q3@A`Fp$0>TpEI7qG!+il)VEzoCt!v#N5qE1$NjP^#>?x*o|Q;a<#|Zo%pIs zTKUdb6!i;j@22zzI=!D-#1$jPbBm&A7ET+h%>N9ppOw2kzV^Ul`$&}UU7{4)id}i* z6yyMcyWt&Ra}!6USdL%k0;!0bP!EWb3I!`Oz|P9qp74+3kN8bK?gxJ#huKE)q6*`Q z0KD1SBsegksnWTvg*qU@Yh1}aX#E$oVY*|R3l)?%G;iNcI>k{iSmoTC=}lm8|wtI&EG46i^k!!Vk@pQ2x2>tDCJS{ zSkyo>Ohw+ddS>0hYe|$}{Y4R_A;Q^=twf0Vy|p5fP=@ME+mn>`v08SQk{3c3Llk-2 z-cN5nGAyh8&1JQPeub!-c$VS~9cE5NKS@a0RIqiL3elaI9_%0TBPf4{ByzTW9xwTH zN4b^X{oM++cMzT+k$ZNq59xQQ6+xQPgwY(d+Q%B}ZJL_(4PZNpxTdd(l-2d$i~lTo&NM+CpoAjF*1Y^h%i3kc9gycO?KKaPMd-9 zL$)i!BT=RDYOa&cqMDDtx}!xqsinmGS|mj+@gUVd$Bl8Ud8cJjZ5Y*HoT88o;dbEH zAq~NpLlm7v{WhMJ`B8cqR^oe!iK;qW5dvHq+8hAhX$Z35o~dQJb~WKIrw3UGoNvt7 zVZS52wH)#!O=>v9>@g(Kc~p1m z8#g{7rHphbrO@+2yi(I!0XOlQ&mo-&6G9Q&I4cTBg6mhQ6i%g0i#0^id2~wYc8`1_ z)4jmYbSbV=jU!5<(?z&UH{aX!X1OcQK|$7P1}~{-7^kbE1O`VvAZ6-BC(^)|_CNgG za5-aDN_tJ%m@D9K*^YPI1!Lc|D%L44?OWS)%5bZMIF)}Z=`(o9aI2JPHd?Vo`G)*k zI1=XXRm>RZ!*zT62U8VW=01v=AC=*&!;$t6W1W6jk^5%c{IHbYo^JnQ!u+fZg-7xq zCLJGYAI00hnR0w+6YDG5er4MHkYAaIw||y#d+ z+R>*G$&>rkp`cQ(ge3D{bt))EnIxJ2s#8O{t(O1Lp(37}h~__ZsE7|!nf!+i6_pa{ zr1BCuR8$rf|1kemr;@mb*&(h&OFTC#?b>SKwvwTw5?M+6XATt^pE)#Sf7Yhjm{UJI zGn;>?LqEnBo$9fzt6dt0dWx zL@fWILq$TPZJqn3Lq~>kX*U~>EmR<#PUADa7|ok34L|3H9rm1X=qUFq+vK7DDo58e8)-Ei))o#-dO z!${wa$J$8RP$u2#$(s)el<_^unm2-^Lp@r{ylo{r(NBJ%2~=E<&3}mPJ3s6|$pPy6 zu;ZI<{d8>kltVwLAX>3m`34Mf>Z50dn+G)3RQF}lO9;1KQNB)nP4m!(##6#g_Bq;> z_?x^MvPa=(jWvBv8*X(aD1e;DQBtv8`DXS5nVr(mct*JTwCdV{noo4dKEtgp1SkA2 zoUK^De9hbsaCQ?aWGl8V->~p&jZIDZ!dlQ0c&N7eMui{e4|fiJHtaU#oAkF2{@gS7 z9yqX}Pjz!m-{DpnJ3Jn**s*+T{eT`()6_(S;Pr!B=bjXG|3C??>xxaw*D3r(JRe_! zcz-G1xU=WyKi@d_z4pfvlwd{`8J&nKfqp6sVAR-m>QH39FnEOUFZ2w35Asd&M z>*pik<{C}lDHGMjc8%s!j88YpeJ9%Xol^X^yxs=62Mul*su@Sk%~-;kgm5Hh6@M+? z+LNr|XVOdJFL;3lFrytfHoRwh>=@>>+(aGbTKh^e(w&!|kBL-C8Upr>jOMkF`5L(bry2`yw@T%dvaL>b6TzoN?58!8t}`dI69k^ef8 z2SPD~=|IDS+b!O$Ag)kr2nW4Iw`yo#LdCNQL3BmTR3oaCeeh!&AF;kvbFsf_s07kB zCIz$an0oW~nVms*9XCk}%#rsHz0*3OazGywt1^uys+9fITQ=NpjD)$wUl^*jSfvs@ zuH^XD4-kf44^>rH5=P~c0)+0YaNUFEyJ9n2e)<{?wIpaVs|?L_05QkXrX)((o1WYA(|07vWiCky(`1dZPz{3nWIV{b zs*WJ+G<8c|lCTFUt5fuaqzjqH*hos)n{T-2-q~89G5!lgbBq=<{S%lvA~<+&B!)TB ztHFAm!Z4==J`ts3HY4Ri22$yxgzMc=B@3qJ$Mg%Q zC~#<~+QR(TePZ#kj3tue9eT+3kF;uuuJl_Xjx-8Zx&j6r)b}zmwh7;irEsxVQt)VW zkyB+$@jc8vWRJ8see}nh*xkdizIvg;$E8K?DsT;q`!L>BuNe0z#Ey3@5!pXm85v+K z&_Xn=#U5#K#*9~&+$+6Y<);@BLUCPmOT}HaTT5OoA!N}!1yQaX_5zcJt|~r(``*yV z%mXPVuX|?wvSY&%=4yXobh$`gft^Gh)Te-~TjpYGx(fY@HPxJ_-2fFzY{Ei|iOK7p z_4YcqpDJOl@fU^-K=BJ$;@zhFHZ_KU;JWec&`bt*2~8clW2zB-5K|a=-Lo&e?U)+v zq;V^;7-DVBwZLoAX3u(WYAk~w#dR|@m8%uf<)w7VDd`YJZuguKYi?oB9U1Sh7g#c! z)VfT}{RH}N-p&rbIvYh-+Z=jP0-zak^kP^khlh#D^Paoy7WD(Amur3X!V?HeaH&)m zX+HZL0@wu&RH)=GQPeK7pThCTtpvkf0iM!zqHlGu4tsl3~{J$&FLAk8W_=OW`@q^3Za_Qsg20)FP@C7EVlI zu2GsumG&$+m>#Bbv>^=BElRo!w<_P=+ny@u8SfffLmcwy>k) zkR}%(N1#oWF zW2y>)12hoFD#ejzq&4Z2apNV;&HmDG9-({wf?BIf@t)>O9?Qr0-cF=}i-j@Kkl;Cv zZcIFZG#6MmEm;2n8Sq56%^2%svu0@uDFvjV?x`9|69j)6-Q}d%RLQH1Z zOcG&6Sv#M;r_EuX2{+_9O`NHf~n?TFKs zNSa&yr6ChgNs7}3)I?Q&(kMXS%y3J?HW2SZR6<=JH#Bx()5H;k$g{0DFp=|mS zlnvL0qG=C}h*}pJhR77AG;+dYtSK9P^|hqAb0yLc6Or+>l|W!_dM=day&^E!9nJxE&>G=}!!&g{4yg#N{s+qPj)o|vhq(Z7}PY?4{jV?HAu8T8;OWV_ZAn1||A`sMC z!))Y-ue8FK^xs0#-0d$7T;i#Gm#9~9p7tj z(b(=OX&q!##s+)Flg7rO5sj|;K>c$c zexph>wtGw32I}X4jQe#WK!VWNuG;q6H)J#S`P&T4dDyY|QJgR|b8ix+?tp86xQno> zDVK~xpb#y(I;9MAp>#rBV_k9A77t3A`~9VXsLr7y(xOeo@j0Q`HdOO)*^d3BNtcZA zfaJ!&8_M(!X>>%5v+BQZ9U*BR@RNr4Q<&go6V7VLpZB3aSBgRfmfDFlRN`<|>;TE* zFk&51GxJI;|b5Bf<%S_t8tx}e<*z~Oj@=IW8rx)G1n^urnPU8t#O_$hf0H<64j zr6cNE>*%gW4VE-B{iP9Q%T8N}L7A1{O;bf0-M=BXaZ96FSzJdW`{7QQLpM62uCxAl zY0E1m%|rguP!+^ckRuS%Y5;_%vw@#L9bMyVoYD}QQblH?D=J1_Q#U%JuD8Cu|AnI@ z&BK1uC~x5kJ=9%l(?o3EbJmCm*;5(fLT3od=u7N{`W0JJ1I;t0G&-Yhuqp@i7$a#O z@sq~xABlR7zo&(UNqg&tq?Q1jP43iMspj zn`p(pmqLNsC&7ddLcF2$#4U|R3}}1JMx}Z-I0Y)o zVY)%E)%S)A6;h0=a0Jst-@qMGZ?9y zaxaS;B|b?VY#7Q8I#`{m;npR+8Qn)@UMaziDU4k4&DJZ2UAVJU^TbM2LyU`dfki4r z8`oPkSO*t~=|eiDfySf=fJmQEmbEgakt?2PMb7>1c}X+NUmCJ+bj;oc7}ZZ7Bi8I4 z#@axFb~ry6`c!+^Rg}_sf@wBP$tdYE$r=~l_GL-)xw?fyt+ngZ=W5-#axQ~WPeU!s2FoX ziG*vjlO0AZPdwQgy5XEV)is^%TNtJ^ouJvT`>@o7y%QI%uX9Fen!1_pGJBFEipCxd zD257|u{pysO1j);P3$*lJ*njxw^|CrVFrb>k!WFQbbxNidEL8#l*W(JG;^wjw1u*f zjuchZSVP0n$O%ueD$6F#l`zk`g(-A2c9Mnzfqp(?u8( zEUF}?F!I5-TcKV5yi>wF=N6_=GejbI8CFSMSYjk^^VOAYSPCqpxlGJj)c68I2xb&< zN6HX}p&2Dzrds{uL%&keGRKv)7>Z#Cq6VyRe)~>`NMIOTl^}L;i$bMBwI{MY4lh#_ zIp902*3BY^N-uN$^+FZPP8iwN8g=AjF<1s=epe{W+(%mU#P^Vb7$D&&9WgYcq|2Sw zR?mM7JB_J(CB3c@8~w-JHK0*1+HCYZ|2!$SWoP_zF^ltTa0hvBH3;Z*_7WnCiMZ$B zHi|r(67Saj&_Ql2<60qnthNsw>_+6tQD^OMI@pb3-FC3-9qdN9XrhqW-*a#q%edZ| zOtdq7-2`;H+Ii|RKWeZWVciL(&F@5T<1@H_FsR!QIrW_UNyTbP847cyLLMSZO2uzv7>Fu)!S)vZtTb&xBubIt%)uEv>Zil4u++zv~ z#)HK@&TBsI&{6KgcHXvwL2PTj{WIql%k5m-@B;A9+n?&Tp!}7IRi&Tjb?2m?8w%ir zacL)SK~DZT3dppZrIUVc8+#iJC;Qy~tj+Qm7vTDZ`Q=DD{ZHXqImzb+4k!DZ57}Ss z@;I!=kjG&>d@~o%)-(Q`P&-3Cq$>)XQKwtJ&kCS_z(|{-Wd`lJrH2mc($>q$S?V4U39wNdBvbIi(@ zevVmWGye0eqo`ZR0t ziOVacn-~40p?FIw%kUGf!)uTv6eAyL;Mu6U>XZi4(|h`8chc(A7}vu5k11;8!1#9%8kK3o<76cXZ-UwNHz1_ zs_Do8k;Z02P|=2kKxXT`PSBC)Ueh&ksz!4&lpCWN&7m`EDa=|c>2i;Cdh1y~OPE*u zgaIzidhQuBSArza=azun61TCVt{2lWiJBp|P!C9JZ4#KN8RhBsS~CvlYp2Gq`lQAX z*(^Gmr`>bVG4~n@@4uA&oNi2dsYeFK;57dBfjkX!z3#p-e*6 z05{>S8caiE9k``>L)N;xzW0$dw7_3O z%>Pi`3-J6RSbW~=#Z3jcNNR*TjwzVOYz9>YHCvv2XGkMo{IGS@F6Hwj&D(y`u*orH zW=R1_zX?a{xywaM(ljpTEe3R>fux!D^mdw2`B4gMH(A3N_dq&RqQEU4bb{?vYwa{%n9Vdfmx%7fH z){Te}TL4LMdYJt#8FC^egU89C#}tgnx*K1{;A>lUWalr=Gr-Kmi| zdO42gcx--vQ5AN%*r6E}4ViSO4y7(4nBhcL?)Y(QpY3W7)H=QAzfO!Zq4O%jIelEt zac|X-{}Ye#hjX2nY{HSLzCQ;HZqTqA#evMuAnEMRWaW^bw4S_Sk6u#FVt?f@MVR(D^mK8{G}gsa zIaC5gZE}%nB@XR8djH~Apg(h|2by`6wRNfIIZ znkPe6zW7<|uKk-@WH?J!VmNd&bZbqG8amA0rzX;nrO-%mj^yWLQFj{S(``l#7BZD1 zZ~UC~`)y-xm2y5=iE?zGS2R{a92|!?0UB{wQGU*)9Ayo_(R-}19y82=^@B8XtolK} z{QF6y`Lv{NwAIDxR^Da!_yxDGxxviK&+{AE#@C%A+fu!v=9`}49V6T57#VMR(tDr! zRySAos>JH~qq1^P_34$280b|Qkn0v+es}sna4apU>jsRq+h8ny;rw7!w=M7gAUCSn zeP$LO;2QVIQZ|!i|L3c`=FIFNB z(IY_(5f`TdX_~h3#&a z3iR3l&2jY;hlrIF*?lk&FLMoCWgZKN2p4fg6N2P$qFyN?=ktkZ`^*ba7W>U$uoJ9p2LL} zuV5m@{SJZ`UA%L~Hl$MQD|<{aXC~A$LoGU@UbaF*uAit?TIRn>Jce2T76UPyi_kcMF#C0$;%{&nQtKTA2^`zePqn*uC~eyjt^A>Q{i9SlU2cIcsQ zl8fRL=MftLsq%Z;?1Z`HKT#ldH!h4b+41T|P&ZnEYIS!MYr4MW0E*H54dTOm8SO zwIgTzrgiDb1MP~*&nsCg_%NdQ;drPQ`)zZ&M~p7O?>O^2#Npnu!bhxGNm=2xipm$i zWv%)1Z{ww$<^IYc<$%IQlBbfoxJRCAg@NeSHuV{&ax~siJ{^)I|6mqyC{QBK0_*p8 zyN!}Kzxa#86;R5vij1My60-Y=Lq@4`WGCY2eKL9#1tlQt+0>3)^4r$ju`Lma^Q*r& zs?+Li#8e1aGdzZucDRAfVS-n1B{A?~s9$i#Gi%+h)|%qTCBI`;jz9J=iSwJkIP%B# z0BtsfE^_*b!v%U}Ixg)H9O^zLIfJUcX0L&I1L7>S4j+ENck(*F`&lwhkUUB+Z6l<# z69>{_d_o$837RgMF&!mc-nHib^|n5eW`&gMM|ZY z7O@h}eML^iJa6XAFoOig><~vTd6D(@#g9BB<^1WV9GzFB;d<4k_`~c~Z{0?_0gM-uLfPPD`mzQHj0~1w`F0(937I4ezt9G;~cb z^mFfWyWmdhOE$!)TL{X9J}@Sayx7VdJMA+mXIQE4lx6Dj+#iCkNYaEMdcG)}<`Kn6 zjyAU)6CiTnQEwv%q^9IpDtY7&tfu;3{wC!N_fw84Zo2Lavsg0^Xb^c@4lkX?%TdjB zt(C^V(6rE4VN}Bv1VcIU$RAq$vMqKPc)p)Fl#QuA6NK|D>a2d3LDJ+7^e0$PmpBB7 zn)AX1X=-kcegm3EoR6&kE?ECc6^=)gy2f)Cz9WcZhK4p-BNck8X`cR=+!BXEKeZEK z^d>iza5(uK{Y|dpBFx9uVduQQo(yNCpW(2ploBy`L-iLH%e!upmVM|Rl}j4JL*4S^ z6xJ*qM_>_?J6>Wfz5MS>WjGi3trQ`F96I9?(7X z2$fV+v()okg&Vlun@XKJLLzsGAG$0@uUd+_<3~;0a5(k^ zkI${IHu|xflyi~4a@dme`thaQc~B$7PaIT(b-B%$5yN;I;z&$~K*v0*;&R7dSR;OH z{$Ao->@SXzGmR#p6Q!fd$K{Y-k_PKCfn%*840FQUO;TPhulKYRy5qRq@t4*$Pd@O2 z#JR*@933xekrNScZN{f^%r$XLofb%E;@HUtLx=!Qm`}iVb(zn`Ix5wG)foEPGmH`kgr@y+c3D z($AB1oh9-YbOB1zMVzyR1S~L zv!@v>=K7r-A3BsHLT>)U&N%jYX*=WCcYNq10kePdNDLUSuu!A2>5OBa zS5Idg`?U9f_pLf;^tl1=YjZoB?jHg`DK8lR-ZtW$z>HW&_pw&WvFD}@k60CseM%>n zAIJVLo+I8%S=>a$4lAATp8L6#Fzh#R4tOu+o=t@Ne;)JQb07b2{CaqLTB}aSyJzeV z{}+BeL@xQR&iVEF(8aIMeWO!;y(WBRR>rT-{g4O0o<^DXaQ8IzYFC(i;g0p>(|!B zpWJ?mGQu(LjL^t>*iDrB5)6m3-&N(Ap3;hd^U9THXIRfxgwtu$qS&0YRgJJ-x%)TP z(Zjn`@Zu3V(5E+@NBJ&Q$zq!FpFOH9SXbT^Mmr?Zh-pi!=-}4I>@QtHjI~L!57|vv%%o z7k#dDFI8dk4o8smixiB!*`5pIRrX9Vm@_TabgtONUR zcb{}K)?YWUal|#qz3+?aqi24KF!ba!?8~JaRqiy;L}@oX0n>64O5VS>qFcsZkT_TQ zi9@*%CM9>T#Fl1|Cv^Z%-C!S@<_DwDxm4V@n`DF%>!}#S}Kg zA=B%)m6?}UHP;TRX=<((b9;k!<~HJp0qWjV9Hw;jDX_2?RHZZRSgHHZ)*m+y`9M{l z@ov==ZYrC|s`o9T9KC(BE;gEi^m z+EV8W{WdQz9oW#sYfRSH#Ta-c^na+>qG3-H*2_v*CD&vqbOGmvRpPRQW?z~;zl2=z zFV>egeQDq4xz4Q@<6JHP|?(u;*}eQUhL6tzgllk zZ=R;nZ`YT$kA54MBg5Tu(X}PMbU63uw`8xRIr^=D4o5u~%A1T#1UOhH+s61a7-XwKMt*{QMdu4z5nw$Oi6fq413}O;?jNf2K1=>VXI3<^A^uj_Y z3Uo^G(yW##jn1e)tk;ILB(+i#{a1>{4+kafAw7!XuD8FT{7yuM7G2_ycVK!%pq$~F zo@t7sGwM(4tAloaOUjwF66J8#(H!Ds3?TFwf5gls7L$u9ZbuBgTrFfmV_3Wd*`_o) zqgq1Umppv3lyl2Ul*1q!?u%-MIkZh4<>`#+&cOEIJnBR_6rO4L^GB{oZCH*@sbQh7 zB9+fdoLg5SjwZj*0#_GOlVtqF;n&%Lrel*+Zep4Z1Or;Miq{>RT*uV#(04ohaizqW z>@N=O5$5j^N7~mceO}7W=p|hz=0I|42XUaxa_5x0COnGc!_bb-s`Eo{v>b7y#JSC1 z9N4Zf&~(!iVkg%my|qJ1Mesp))+G)hGwB4*QG3F1XPI*yaYlspS|fatEP0CCk_$r$ zT`AxS4IwXEi!QczNI`=JE*3$DbFCQ8K}!OjAFNjh3uZ`>l0zOD+WLnpuaXD6-R%Jj zr6Hgr7=U}o+)5Y+T8Pf0E#@x~KD)bG6iNs|*-tz}mtoCKo{~GhAoTZFZkZ?5Om(ZK z!yOZ*V}b14wwtit?ig9%#nP>GFO&jis6^zOb`oT6@|67XsL+%r?@P#T?(nx8YylRB z-fY9f1x;7>+T^-jPt;1w-L+Cg%)CQG>}F^j_;DK_r(>2u_qt9 z8rO`+0SgyRU@y{6cWfzns% zsY}T@nr4%hE50=J$kZPfOPqWB#Szz)TSJhtnJUMKWbZAd9uQ2*P7W$~8LCN%Xv!Tm zM^urPFTN}^Wt%bUNt}EA#1W}UlqOivum{*v-YFh8{&i1_FwwPG#D;{TMDkc6d?bf< zA>dL$lz7y>Oz6#&$#REUo-*pJwIPd& znOsa|8~{7P5JjH&%Fv3l&R(c>deDEJU>s_I39*2f_C7;gk%d5%ee9~W(XNh@%!`f0 znzK}^h@+ceuhnW9t_O?WB)Q2*%{y1uW-`d)`Vq3 z98(%OP^ktrD<1#Sw2uE`mBZR7^)`Y!Cam zI&}O+!|f5b54)Aq5#cBuhd_jy?RzK0em>IZRU$M3I*~@#2lY}OFp=1s%8@(1CbauM zFS7&nM^>U7HVMa}(xo_TA|H2*eNZm%#2wQ@W=Dza1%r}pZtf|2;>Wnqq@^33qqTa} zPaGn=ekh0kHbG|n?K>UtW7_(qIM$pp8!IE1d_$;o@kzGnJn1J6XBB7}hr*>f<|N~u zJ@z@`AUc2Ol^{+_COTxHR5P|K&*lk{kxRZYbZo^E`wqfW?zJlPI&22f{$XX>*XI1B z(GdSmd?1_-m3#4l#K#1IW}Pb|mwZ#`)cWknIn zr5p`ERe1pB9i{^W_m)pd6Yp9FXq2mgq+1 za1G)i9&omQv+~EcgkB%H+bL4c9Dn6#e2y;P!Mo;9zojCzBX`bR*EZWqL9P(|Un<^61 zWI8YSn+|6@4;&Ayw?OmrKGbWt@EyPMVbY-umH1fw&Lfub4#(w9_ zJU?kv=hEdV?sUm?(6_fVn)Inu*Ch>OZ8SWeLtaf_$92N2{PFFf72E#hCh6uyKWWJG zm>o`SO4l|SnZUN{JSassOwi|!GWArK&(YK)>LzizO$>K5Plv4B@zl^K2mbb;604Vd z6D#s5RgEP`ZUl*^H>N9vn{yR%;Peq#us%q_EGLK2$Qj=ex~J#4wljX&tr+9jWHYfX z*_ve1p0A@oFZ3J;Tsg*F%S6usd*df`k%2u`Xfy-}NONcCwwJ1QRFiXlzR7vF-Q;YY z_sy)3Uz3vuIA?w&duftK!TYTe9Wy_0cl=v(zR4NqZC741KY}$m@42LYiJF|Rv^P0( zK$hO{13-c_IcKD9-&=k0S6zK^5>kE)zk}qz8kZpS20k$t z#kF>DkaYQqB*dUxJ{DAc?20)(O3RlNa`znsK-y_}-vuMRlhGLZX??V>t%S0TpE-!jntMYnR~ zVC|*sdRxLJh~=KTf!ZManiqlFJUxdgFsdAWYR{UrlsSR|)v+l>Len4zYr$@yv9ccS6k zy9acsFsa|%={*Si%F5p}kozIedk_QxZ4$ZOGXUv~uw=XP3KR9}} z=mgNCN1A&>oBw+KE>g}r{>p(~pe84jF{ZrI+wzu%%7+T?lr^0>7^Qa#gwTITD5%=DZo!&<@brqV$5*ewlJ6}T^=W=Qaz4QO=3q;iu8Q9FtX5>q1&!4_=N141?m5i_ApOL-$>iwi)Mvdd>QvJr(ZJ z(2Nxijg>eb`HKTD8i&v2FD|&j;PqD*9M+{OPDT^8mA>N6Hfe)zG_ zIcN91S!VOGzuB<4p@Gn7V5}-v54>*9*y6x==auE1NJF_^Uu2A#T4y=Zu=d{W1_NQZnOhh_LDjhFiikyIc8(xTBh0MOF?()pzXbYI_eyny zS}MZOR-;Tvl%$%Zx4U5s6D&YT8ZOm{$4#kRu@M6x&8vFUF_GrU(C$^qgQc3KeyYI} z!?c0+!YvV+W(egc4qbd5u1;zdIU58dI*xEJG%~Kxh+}faPlYyI@RB_(>N7vHu>(ka zmdznze|lc=#E7Y&A-Mu6r`crb8__0&Y6D{hcWjCySNwG7;Qfc%FMWkkw1zm^!3Fn9NU_czYJYrXikmvBmF?r)>Li^3Qxv%`qm;Tb|NU$BLZYn{(^|l+m zy%M4?KjJvU$=JEKN&^(?BxDCOV~8mxem1n=j?@?OH(&W1jygVs6SQ{4M8XC=&u}sZ zLUg%)hv6WNnuC<^Q7&h;8Qs_e;hqb9zoPsViSxC;IE1nA{4pCsTFu+`6o+Ce0k|gF zxx^u3z(5GB$%PQdbbt0pxH+Mx&z||6#QA0=;^<{fbV;CI(CYBK%cv}e^O4RDygjFO zMC8`HaKsdIj^c$+#A3}2o%(KNsZ3{?pXp$N9Gb9cu~4FHW<+>v$4(jG=(-{t23J7I zpi4+|)D>^V<&K{ZP2TO}UnR}Ae$qg?pxaF%pqCG!BSxrsnGW~-$#d}7ZbK$8#D$8T zg)=qZG(}v9G%th>3XlGWmg+mdrIKzSX6g0zEEK!~?`_4+dfv{#|6E}@^U}$`X$`|I zz@kmsEpaa5%nMZ~9@t3A`QA@Cx&)!y&ZH*zcc0e}VmY!84r|vd*19dF=J-P6_-PP zDb)4-b5E&5IX{-x#c8lTVK>-X9+|h)s`%@Xjn#c>dRO;3rE;f14Nc9Jj5?{QZ?5c9 zJ#b*d%@`G2x2)c-GIQb?4-t zwfxHk2tTN@@}IQ>Ybv|a4L+WdK z9>Raw|F9!_Bs%_6b7OUFeR%L0-FK@T$d6SLzV238TVJ`ypoYFf2G;zoyplf#@iRN^ z96$|q(;og+YFJ*Vh6S@122sPN6^Cu}5YPIhv~HM{*TbC87omq!{~zcfVe3KAZ5=cY zsjshT4BPq&q>C-SjJ+aV{934sIUg?$qKnMMdoKKnXZ==McYlX27Qglpy12z!ZDq7s zPNHWHbW_{^SM{!LswuD)%6YbqY@OIjvj3@yIRnj2m3vk;)eP*HQ&^RK31Fu_LmG4c z6vUP`{He6J^!IzA{uaNuB#8c2d~!hfr#x##Y2Do&`dfC@QuKG@|C#pdofL-|5o>z~P=Kd(uTx;v+K{R*I+Jm3|m}mW2T6brM=H|@$63ty# zM9npWyQ!xPu1fz{V0A=uE7e+6&Jb(0Wvd1@Gz{J;fZDz}V}t#qwwAI2A3kQ%*Fn^_ z@Q#hQ_>5-_E32E)p|+*3EkkV+{!i3K7;US~r5fAhnmF$8ZubA}qa&p}uc*yk|}QX-z$A&Zy}*u(o&4!DloNXsGAfWY4DB=9(UZfmq$Yrm1Ih zLt~!-J*%2)>YExG3v7+SueQ?Kb&+#&`r&Z(kmiQ|jn#cS^c29v?)_=qMN-p&!j7(PtgY@n zuqIqpb9zmmAsIK$;sWn+McX?z`)*mAN0S z>WqzVUuPes>(t8dX-;Js*{fyI}=d z?{-kl8ND0WLI>72*EBW`8QdH|)??Rwq*k)F7RtJ8{vSbP9d%Xr^fo+ebQx2Cy06n% z+rP1)Kd1C*{fF#iU16=sG>Hvvu`b=N_PCA9LtQQ`%MIntMa#>+eCiATp}Zb+G@4eL zR%g?=ibqG&jmh?>ItU?UG2Z5x|IooFDx6tC&#~N<2$$}^5%YHr)0DeCR?GJ+-DBmlrl{&Ka17L zMJprci*(G@bt;F-tg z3=)^q{cr;yzaX8^(K;UMq@x*T**PDrO$SasTA{IbHjy_E(>alsUs)iJV|($TJr&;e zdCrl%hH5*Zqjk)@?wpR+W_M0HT22RTF#lij(ex4DWy)H#&540b6Yz+b$k z^0s~6IU}uGdDk47%qE?cq857=ze&w1_^cB?THA=-c0O7uhNYSP9z%Kcp=&5FoeRFv zDH)9omyE7#C~xkEJjiHp4{dIE4yTi5#yyVzfsAJVR3RDd!8`6gaKkl93mI#0(z2O7 zE?IoAUdee;S;aYCGavp>s=X&zb5oCpc-=GOw zw|blw>t0gV(0i9X_*;(w&CP?Gs;a8`*ESCr(mULzVNg}wkb32l(L|NIaGFb_$$Vm! zHTJN$Shg>>ajEmidiTrpj`i(bok>;4)78n;0oGyG@oNsOZe{>y@9`nt<-6I?>iWLD zdo+=Dl799W+%T}V&lx>Vi&kt>zHZxpJh=Ldh9S+}O1mE&>)!pe^357+P90L)STl&y zbW^xrW5by>_2K5zo4ak^|DZkx98h=s-a`i+m^re3-=2L>`Nsjf?!D)pmGl3<&aPc2 zVi*W29ipJ1qP`EfB)&ZChT`~<)d}(87^rAzx0%s)Gqc(`fcx~1 zC_ZW%Wqvve?A!E*LW=z)8K%>iM$;}TGM-MoBm2N~Ja9Vn*EhdRElc4RBECy;zp1UMm5~>Vu;98Cn?kkJ&OCK5{9IHeT$+t7o zxs3)rlEUah4%nkXLzg-1C>Q|El=F*Fv?PnJlpnp?#gM;>c*Bz zb!(kT?kvrK`;^n>WA+ zt~>5(+)%W364!CiE|@eF0!n7fR$zub^Pw4s4Njf{0UfA@-p7I!&Bm2GT|;`x93gq} E1t7HVMgRZ+ diff --git a/wandb/run-20231023_091029-txn4i28h/files/conda-environment.yaml b/wandb/run-20231023_091029-txn4i28h/files/conda-environment.yaml deleted file mode 100644 index cfced9e..0000000 --- a/wandb/run-20231023_091029-txn4i28h/files/conda-environment.yaml +++ /dev/null @@ -1,297 +0,0 @@ -name: eye -channels: - - pytorch - - defaults -dependencies: - - _libgcc_mutex=0.1=main - - _openmp_mutex=5.1=1_gnu - - blas=1.0=mkl - - brotlipy=0.7.0=py39h27cfd23_1003 - - bzip2=1.0.8=h7b6447c_0 - - ca-certificates=2023.01.10=h06a4308_0 - - certifi=2022.12.7=py39h06a4308_0 - - cffi=1.15.1=py39h5eee18b_3 - - charset-normalizer=2.0.4=pyhd3eb1b0_0 - - cryptography=39.0.1=py39h9ce1e76_0 - - cudatoolkit=11.3.1=h2bc3f7f_2 - - ffmpeg=4.3=hf484d3e_0 - - flit-core=3.8.0=py39h06a4308_0 - - freetype=2.12.1=h4a9f257_0 - - giflib=5.2.1=h5eee18b_3 - - gmp=6.2.1=h295c915_3 - - gnutls=3.6.15=he1e5248_0 - - idna=3.4=py39h06a4308_0 - - intel-openmp=2021.4.0=h06a4308_3561 - - jpeg=9e=h5eee18b_1 - - lame=3.100=h7b6447c_0 - - lcms2=2.12=h3be6417_0 - - ld_impl_linux-64=2.38=h1181459_1 - - lerc=3.0=h295c915_0 - - libdeflate=1.17=h5eee18b_0 - - libffi=3.4.2=h6a678d5_6 - - libgcc-ng=11.2.0=h1234567_1 - - libgomp=11.2.0=h1234567_1 - - libiconv=1.16=h7f8727e_2 - - libidn2=2.3.2=h7f8727e_0 - - libpng=1.6.39=h5eee18b_0 - - libstdcxx-ng=11.2.0=h1234567_1 - - libtasn1=4.19.0=h5eee18b_0 - - libtiff=4.5.0=h6a678d5_2 - - libunistring=0.9.10=h27cfd23_0 - - libwebp=1.2.4=h11a3e52_1 - - libwebp-base=1.2.4=h5eee18b_1 - - lz4-c=1.9.4=h6a678d5_0 - - mkl=2021.4.0=h06a4308_640 - - mkl-service=2.4.0=py39h7f8727e_0 - - mkl_fft=1.3.1=py39hd3c417c_0 - - mkl_random=1.2.2=py39h51133e4_0 - - ncurses=6.4=h6a678d5_0 - - nettle=3.7.3=hbbd107a_1 - - numpy=1.23.5=py39h14f4228_0 - - numpy-base=1.23.5=py39h31eccc5_0 - - openh264=2.1.1=h4ff587b_0 - - openssl=1.1.1t=h7f8727e_0 - - pillow=9.4.0=py39h6a678d5_0 - - pip=23.0.1=py39h06a4308_0 - - pycparser=2.21=pyhd3eb1b0_0 - - pyopenssl=23.0.0=py39h06a4308_0 - - pysocks=1.7.1=py39h06a4308_0 - - python=3.9.16=h7a1cb2a_2 - - pytorch-mutex=1.0=cuda - - readline=8.2=h5eee18b_0 - - requests=2.28.1=py39h06a4308_1 - - setuptools=65.6.3=py39h06a4308_0 - - six=1.16.0=pyhd3eb1b0_1 - - sqlite=3.41.1=h5eee18b_0 - - tk=8.6.12=h1ccaba5_0 - - torchaudio=0.12.1=py39_cu113 - - torchvision=0.13.1=py39_cu113 - - typing_extensions=4.4.0=py39h06a4308_0 - - urllib3=1.26.15=py39h06a4308_0 - - wheel=0.38.4=py39h06a4308_0 - - xz=5.2.10=h5eee18b_1 - - zlib=1.2.13=h5eee18b_0 - - zstd=1.5.4=hc292b87_0 - - pip: - - accelerate==0.20.3 - - aiofiles==23.1.0 - - aiohttp==3.8.4 - - aiosignal==1.3.1 - - altair==4.2.2 - - antlr4-python3-runtime==4.9.3 - - anyio==3.6.2 - - appdirs==1.4.4 - - argon2-cffi==21.3.0 - - argon2-cffi-bindings==21.2.0 - - arrow==1.2.3 - - asttokens==2.2.1 - - async-timeout==4.0.2 - - attrs==22.2.0 - - backcall==0.2.0 - - beautifulsoup4==4.12.2 - - bitsandbytes==0.37.0 - - bleach==6.0.0 - - blis==0.7.9 - - braceexpand==0.1.7 - - catalogue==2.0.8 - - cchardet==2.1.7 - - chardet==5.1.0 - - click==8.1.3 - - cloudpickle==2.2.1 - - cmake==3.26.1 - - comm==0.1.3 - - confection==0.0.4 - - contourpy==1.0.7 - - cycler==0.11.0 - - cymem==2.0.7 - - datasets==2.13.1 - - debugpy==1.6.7 - - decorator==5.1.1 - - decord==0.6.0 - - defusedxml==0.7.1 - - dill==0.3.6 - - docker-pycreds==0.4.0 - - einops==0.6.0 - - entrypoints==0.4 - - executing==1.2.0 - - fairscale==0.4.13 - - fastapi==0.95.0 - - fastjsonschema==2.16.3 - - ffmpy==0.3.0 - - filelock==3.9.0 - - fire==0.5.0 - - fonttools==4.38.0 - - fqdn==1.5.1 - - frozenlist==1.3.3 - - fsspec==2023.4.0 - - gitdb==4.0.10 - - gitpython==3.1.31 - - gradio==3.24.1 - - gradio-client==0.0.8 - - h11==0.14.0 - - httpcore==0.16.3 - - httpx==0.23.3 - - huggingface-hub==0.16.4 - - imageio==2.31.1 - - importlib-metadata==6.2.0 - - importlib-resources==5.12.0 - - iopath==0.1.10 - - ipykernel==6.22.0 - - ipython==8.12.0 - - ipython-genutils==0.2.0 - - isoduration==20.11.0 - - jedi==0.18.2 - - jinja2==3.1.2 - - joblib==1.2.0 - - jsonpointer==2.3 - - jsonschema==4.17.3 - - jupyter-client==8.1.0 - - jupyter-core==5.3.0 - - jupyter-events==0.6.3 - - jupyter-server==2.5.0 - - jupyter-server-terminals==0.4.4 - - jupyterlab-pygments==0.2.2 - - kiwisolver==1.4.4 - - langcodes==3.3.0 - - lazy-loader==0.2 - - linkify-it-py==2.0.0 - - lit==16.0.0 - - llvmlite==0.39.1 - - loralib==0.1.1 - - markdown-it-py==2.2.0 - - markupsafe==2.1.2 - - matplotlib==3.7.0 - - matplotlib-inline==0.1.6 - - mdit-py-plugins==0.3.3 - - mdurl==0.1.2 - - mistune==2.0.5 - - mpmath==1.3.0 - - multidict==6.0.4 - - multiprocess==0.70.14 - - murmurhash==1.0.9 - - nbclassic==0.5.5 - - nbclient==0.7.3 - - nbconvert==7.3.0 - - nbformat==5.8.0 - - nest-asyncio==1.5.6 - - networkx==3.1 - - nltk==3.8.1 - - notebook==6.5.4 - - notebook-shim==0.2.2 - - numba==0.56.4 - - nvidia-cublas-cu11==11.10.3.66 - - nvidia-cuda-cupti-cu11==11.7.101 - - nvidia-cuda-nvrtc-cu11==11.7.99 - - nvidia-cuda-runtime-cu11==11.7.99 - - nvidia-cudnn-cu11==8.5.0.96 - - nvidia-cufft-cu11==10.9.0.58 - - nvidia-curand-cu11==10.2.10.91 - - nvidia-cusolver-cu11==11.4.0.1 - - nvidia-cusparse-cu11==11.7.4.91 - - nvidia-nccl-cu11==2.14.3 - - nvidia-nvtx-cu11==11.7.91 - - omegaconf==2.3.0 - - openai==0.27.0 - - opencv-python==4.7.0.72 - - orjson==3.8.9 - - packaging==23.0 - - pandas==2.0.0 - - pandocfilters==1.5.0 - - parso==0.8.3 - - pathlib==1.0.1 - - pathtools==0.1.2 - - pathy==0.10.1 - - peft==0.2.0 - - pexpect==4.8.0 - - pickleshare==0.7.5 - - platformdirs==3.2.0 - - policies==0.4.2 - - portalocker==2.7.0 - - preshed==3.0.8 - - progressbar2==4.2.0 - - prometheus-client==0.16.0 - - prompt-toolkit==3.0.38 - - protobuf==4.22.1 - - psutil==5.9.4 - - ptyprocess==0.7.0 - - pure-eval==0.2.2 - - pyarrow==12.0.1 - - pycocoevalcap==1.2 - - pycocotools==2.0.6 - - pycryptodome==3.17 - - pydantic==1.10.7 - - pydub==0.25.1 - - pygments==2.14.0 - - pynndescent==0.5.8 - - pyparsing==3.0.9 - - pyrsistent==0.19.3 - - python-dateutil==2.8.2 - - python-json-logger==2.0.7 - - python-multipart==0.0.6 - - python-utils==3.7.0 - - pytz==2023.3 - - pywavelets==1.4.1 - - pyyaml==6.0 - - pyzmq==25.0.2 - - regex==2022.10.31 - - rfc3339-validator==0.1.4 - - rfc3986==1.5.0 - - rfc3986-validator==0.1.1 - - safetensors==0.3.1 - - scikit-image==0.21.0 - - scikit-learn==1.2.2 - - scipy==1.10.1 - - seaborn==0.12.2 - - semantic-version==2.10.0 - - send2trash==1.8.0 - - sentence-transformers==2.2.2 - - sentencepiece==0.1.97 - - sentry-sdk==1.19.1 - - setproctitle==1.3.2 - - smart-open==6.3.0 - - smmap==5.0.0 - - sniffio==1.3.0 - - soupsieve==2.4 - - spacy==3.5.1 - - spacy-legacy==3.0.12 - - spacy-loggers==1.0.4 - - srsly==2.4.6 - - stack-data==0.6.2 - - starlette==0.26.1 - - submitit==1.4.5 - - sympy==1.11.1 - - tenacity==8.2.2 - - termcolor==2.3.0 - - terminado==0.17.1 - - thinc==8.1.9 - - threadpoolctl==3.1.0 - - tifffile==2023.4.12 - - timm==0.6.13 - - tinycss2==1.2.1 - - tokenizers==0.13.3 - - toolz==0.12.0 - - torch==2.0.0 - - tornado==6.2 - - tqdm==4.64.1 - - traitlets==5.9.0 - - transformers==4.31.0 - - triton==2.0.0 - - typer==0.7.0 - - tzdata==2023.3 - - uc-micro-py==1.0.1 - - umap-learn==0.5.3 - - uri-template==1.2.0 - - uvicorn==0.21.1 - - visual-genome==1.1.1 - - wandb==0.14.2 - - wasabi==1.1.1 - - wcwidth==0.2.6 - - webcolors==1.13 - - webdataset==0.2.48 - - webencodings==0.5.1 - - websocket-client==1.5.1 - - websockets==11.0.1 - - xxhash==3.2.0 - - yarl==1.8.2 - - zipp==3.14.0 -prefix: /home/chenj0g/anaconda3/envs/eye diff --git a/wandb/run-20231023_091029-txn4i28h/files/config.yaml b/wandb/run-20231023_091029-txn4i28h/files/config.yaml deleted file mode 100644 index b1eeb43..0000000 --- a/wandb/run-20231023_091029-txn4i28h/files/config.yaml +++ /dev/null @@ -1,38 +0,0 @@ -wandb_version: 1 - -_wandb: - desc: null - value: - python_version: 3.9.16 - cli_version: 0.14.2 - framework: huggingface - huggingface_version: 4.31.0 - is_jupyter_run: false - is_kaggle_kernel: false - start_time: 1698041430.035569 - t: - 1: - - 1 - - 11 - - 41 - - 49 - - 55 - - 63 - - 71 - 2: - - 1 - - 11 - - 41 - - 49 - - 55 - - 63 - - 71 - 3: - - 1 - - 13 - - 23 - 4: 3.9.16 - 5: 0.14.2 - 6: 4.31.0 - 8: - - 5 diff --git a/wandb/run-20231023_091029-txn4i28h/files/output.log b/wandb/run-20231023_091029-txn4i28h/files/output.log deleted file mode 100644 index dccb2b0..0000000 --- a/wandb/run-20231023_091029-txn4i28h/files/output.log +++ /dev/null @@ -1,2 +0,0 @@ -/home/chenj0g/anaconda3/envs/eye/lib/python3.9/site-packages/torch/utils/checkpoint.py:31: UserWarning: None of the inputs have requires_grad=True. Gradients will be None - warnings.warn("None of the inputs have requires_grad=True. Gradients will be None") \ No newline at end of file diff --git a/wandb/run-20231023_091029-txn4i28h/files/requirements.txt b/wandb/run-20231023_091029-txn4i28h/files/requirements.txt deleted file mode 100644 index 75bc393..0000000 --- a/wandb/run-20231023_091029-txn4i28h/files/requirements.txt +++ /dev/null @@ -1,246 +0,0 @@ -accelerate==0.20.3 -aiofiles==23.2.1 -aiohttp==3.8.4 -aiosignal==1.3.1 -altair==5.1.2 -antlr4-python3-runtime==4.9.3 -anyio==3.6.2 -appdirs==1.4.4 -argon2-cffi-bindings==21.2.0 -argon2-cffi==21.3.0 -arrow==1.2.3 -asttokens==2.2.1 -async-timeout==4.0.2 -attrs==22.2.0 -backcall==0.2.0 -beautifulsoup4==4.12.2 -bitsandbytes==0.37.0 -bleach==6.0.0 -blis==0.7.9 -braceexpand==0.1.7 -brotlipy==0.7.0 -catalogue==2.0.8 -cchardet==2.1.7 -certifi==2022.12.7 -cffi==1.15.1 -chardet==5.1.0 -charset-normalizer==2.0.4 -click==8.1.3 -cloudpickle==2.2.1 -cmake==3.26.1 -comm==0.1.3 -confection==0.0.4 -contourpy==1.0.7 -cryptography==39.0.1 -cycler==0.11.0 -cymem==2.0.7 -datasets==2.13.1 -debugpy==1.6.7 -decorator==5.1.1 -decord==0.6.0 -defusedxml==0.7.1 -dill==0.3.6 -docker-pycreds==0.4.0 -einops==0.6.0 -entrypoints==0.4 -executing==1.2.0 -fairscale==0.4.13 -fastapi==0.95.0 -fastjsonschema==2.16.3 -ffmpy==0.3.1 -filelock==3.9.0 -fire==0.5.0 -flit-core==3.8.0 -fonttools==4.38.0 -fqdn==1.5.1 -frozenlist==1.3.3 -fsspec==2023.9.2 -gitdb==4.0.10 -gitpython==3.1.31 -gradio-client==0.6.0 -gradio==3.24.1 -h11==0.14.0 -httpcore==0.18.0 -httpx==0.25.0 -huggingface-hub==0.18.0 -idna==3.4 -imageio==2.31.1 -importlib-metadata==6.2.0 -importlib-resources==5.12.0 -iopath==0.1.10 -ipykernel==6.22.0 -ipython-genutils==0.2.0 -ipython==8.12.0 -isoduration==20.11.0 -jedi==0.18.2 -jinja2==3.1.2 -joblib==1.2.0 -jsonpointer==2.3 -jsonschema==4.17.3 -jupyter-client==8.1.0 -jupyter-core==5.3.0 -jupyter-events==0.6.3 -jupyter-server-terminals==0.4.4 -jupyter-server==2.5.0 -jupyterlab-pygments==0.2.2 -kiwisolver==1.4.4 -langcodes==3.3.0 -lazy-loader==0.2 -linkify-it-py==2.0.2 -lit==16.0.0 -llvmlite==0.39.1 -loralib==0.1.1 -markdown-it-py==2.2.0 -markupsafe==2.1.2 -matplotlib-inline==0.1.6 -matplotlib==3.7.0 -mdit-py-plugins==0.3.3 -mdurl==0.1.2 -mistune==2.0.5 -mkl-fft==1.3.1 -mkl-random==1.2.2 -mkl-service==2.4.0 -mpmath==1.3.0 -multidict==6.0.4 -multiprocess==0.70.14 -murmurhash==1.0.9 -nbclassic==0.5.5 -nbclient==0.7.3 -nbconvert==7.3.0 -nbformat==5.8.0 -nest-asyncio==1.5.6 -networkx==3.1 -nltk==3.8.1 -notebook-shim==0.2.2 -notebook==6.5.4 -numba==0.56.4 -numpy==1.23.5 -nvidia-cublas-cu11==11.10.3.66 -nvidia-cuda-cupti-cu11==11.7.101 -nvidia-cuda-nvrtc-cu11==11.7.99 -nvidia-cuda-runtime-cu11==11.7.99 -nvidia-cudnn-cu11==8.5.0.96 -nvidia-cufft-cu11==10.9.0.58 -nvidia-curand-cu11==10.2.10.91 -nvidia-cusolver-cu11==11.4.0.1 -nvidia-cusparse-cu11==11.7.4.91 -nvidia-nccl-cu11==2.14.3 -nvidia-nvtx-cu11==11.7.91 -omegaconf==2.3.0 -openai==0.27.0 -opencv-python==4.7.0.72 -orjson==3.9.8 -packaging==23.0 -pandas==2.0.0 -pandocfilters==1.5.0 -parso==0.8.3 -pathlib==1.0.1 -pathtools==0.1.2 -pathy==0.10.1 -peft==0.2.0 -pexpect==4.8.0 -pickleshare==0.7.5 -pillow==9.4.0 -pip==23.0.1 -platformdirs==3.2.0 -policies==0.4.2 -portalocker==2.7.0 -preshed==3.0.8 -progressbar2==4.2.0 -prometheus-client==0.16.0 -prompt-toolkit==3.0.38 -protobuf==4.22.1 -psutil==5.9.4 -ptyprocess==0.7.0 -pure-eval==0.2.2 -pyarrow==12.0.1 -pycocoevalcap==1.2 -pycocotools==2.0.6 -pycparser==2.21 -pycryptodome==3.17 -pydantic==1.10.7 -pydub==0.25.1 -pygments==2.14.0 -pynndescent==0.5.8 -pyopenssl==23.0.0 -pyparsing==3.0.9 -pyrsistent==0.19.3 -pysocks==1.7.1 -python-dateutil==2.8.2 -python-json-logger==2.0.7 -python-multipart==0.0.6 -python-utils==3.7.0 -pytz==2023.3 -pywavelets==1.4.1 -pyyaml==6.0 -pyzmq==25.0.2 -regex==2022.10.31 -requests==2.28.1 -rfc3339-validator==0.1.4 -rfc3986-validator==0.1.1 -rfc3986==1.5.0 -rpe-index==1.2.0 -safetensors==0.3.1 -scikit-image==0.21.0 -scikit-learn==1.2.2 -scipy==1.10.1 -seaborn==0.12.2 -semantic-version==2.10.0 -send2trash==1.8.0 -sentence-transformers==2.2.2 -sentencepiece==0.1.97 -sentry-sdk==1.19.1 -setproctitle==1.3.2 -setuptools==65.6.3 -six==1.16.0 -smart-open==6.3.0 -smmap==5.0.0 -sniffio==1.3.0 -soupsieve==2.4 -spacy-legacy==3.0.12 -spacy-loggers==1.0.4 -spacy==3.5.1 -srsly==2.4.6 -stack-data==0.6.2 -starlette==0.26.1 -submitit==1.4.5 -sympy==1.11.1 -tenacity==8.2.2 -termcolor==2.3.0 -terminado==0.17.1 -thinc==8.1.9 -threadpoolctl==3.1.0 -tifffile==2023.4.12 -timm==0.6.13 -tinycss2==1.2.1 -tokenizers==0.13.3 -toolz==0.12.0 -torch==2.0.0 -torchaudio==0.12.1 -torchvision==0.13.1 -tornado==6.2 -tqdm==4.64.1 -traitlets==5.9.0 -transformers==4.31.0 -triton==2.0.0 -typer==0.7.0 -typing-extensions==4.4.0 -tzdata==2023.3 -uc-micro-py==1.0.2 -umap-learn==0.5.3 -uri-template==1.2.0 -urllib3==1.26.15 -uvicorn==0.21.1 -visual-genome==1.1.1 -wandb==0.14.2 -wasabi==1.1.1 -wcwidth==0.2.6 -webcolors==1.13 -webdataset==0.2.48 -webencodings==0.5.1 -websocket-client==1.5.1 -websockets==11.0.1 -wheel==0.38.4 -xxhash==3.2.0 -yarl==1.8.2 -zipp==3.14.0 \ No newline at end of file diff --git a/wandb/run-20231023_091029-txn4i28h/files/wandb-metadata.json b/wandb/run-20231023_091029-txn4i28h/files/wandb-metadata.json deleted file mode 100644 index cfa596e..0000000 --- a/wandb/run-20231023_091029-txn4i28h/files/wandb-metadata.json +++ /dev/null @@ -1,374 +0,0 @@ -{ - "os": "Linux-3.10.0-1160.76.1.el7.x86_64-x86_64-with-glibc2.17", - "python": "3.9.16", - "heartbeatAt": "2023-10-23T06:10:31.384548", - "startedAt": "2023-10-23T06:10:29.992874", - "docker": null, - "cuda": null, - "args": [ - "--job_name", - "minigpt4_v2_test", - "--cfg-path", - "train_configs/minigpt_v2_finetune.yaml" - ], - "state": "running", - "program": "/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/train.py", - "codePath": "train.py", - "git": { - "remote": "https://github.com/junchen14/MiniGPT-4_finetune", - "commit": "5c3ec8bb730b2e9df23b4bb3eb24108f9c04a8d6" - }, - "email": "jun.chen@kaust.edu.sa", - "root": "/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune", - "host": "gpu201-16-l", - "username": "chenj0g", - "executable": "/home/chenj0g/anaconda3/envs/eye/bin/python", - "cpu_count": 64, - "cpu_count_logical": 64, - "cpu_freq": { - "current": 1996.2770000000016, - "min": 0.0, - "max": 0.0 - }, - "cpu_freq_per_core": [ - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - } - ], - "disk": { - "total": 251.85773849487305, - "used": 7.905696868896484 - }, - "gpu": "NVIDIA A100-SXM4-80GB", - "gpu_count": 2, - "gpu_devices": [ - { - "name": "NVIDIA A100-SXM4-80GB", - "memory_total": 85899345920 - }, - { - "name": "NVIDIA A100-SXM4-80GB", - "memory_total": 85899345920 - } - ], - "memory": { - "total": 503.7154769897461 - } -} diff --git a/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json b/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json deleted file mode 100644 index 012577d..0000000 --- a/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json +++ /dev/null @@ -1 +0,0 @@ -{"epoch": 0, "loss": 1.071101188659668, "_timestamp": 1698041570.1773224, "_runtime": 140.1417534351349, "_step": 258, "_wandb": {"runtime": 139}} \ No newline at end of file diff --git a/wandb/run-20231023_091029-txn4i28h/logs/debug-internal.log b/wandb/run-20231023_091029-txn4i28h/logs/debug-internal.log deleted file mode 100644 index 809670b..0000000 --- a/wandb/run-20231023_091029-txn4i28h/logs/debug-internal.log +++ /dev/null @@ -1,1375 +0,0 @@ -2023-10-23 09:10:30,034 INFO StreamThr :54597 [internal.py:wandb_internal():86] W&B internal server running at pid: 54597, started at: 2023-10-23 09:10:30.033566 -2023-10-23 09:10:30,034 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: status -2023-10-23 09:10:30,037 INFO WriterThread:54597 [datastore.py:open_for_write():85] open: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/run-txn4i28h.wandb -2023-10-23 09:10:30,038 DEBUG SenderThread:54597 [sender.py:send():375] send: header -2023-10-23 09:10:30,447 DEBUG SenderThread:54597 [sender.py:send():375] send: run -2023-10-23 09:10:31,051 INFO SenderThread:54597 [dir_watcher.py:__init__():219] watching files in: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files -2023-10-23 09:10:31,051 INFO SenderThread:54597 [sender.py:_start_run_threads():1124] run started: txn4i28h with start time 1698041430.035569 -2023-10-23 09:10:31,051 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:31,052 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:31,053 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: check_version -2023-10-23 09:10:31,053 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: check_version -2023-10-23 09:10:31,362 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: run_start -2023-10-23 09:10:31,367 DEBUG HandlerThread:54597 [system_info.py:__init__():31] System info init -2023-10-23 09:10:31,367 DEBUG HandlerThread:54597 [system_info.py:__init__():46] System info init done -2023-10-23 09:10:31,367 INFO HandlerThread:54597 [system_monitor.py:start():181] Starting system monitor -2023-10-23 09:10:31,367 INFO SystemMonitor:54597 [system_monitor.py:_start():145] Starting system asset monitoring threads -2023-10-23 09:10:31,368 INFO HandlerThread:54597 [system_monitor.py:probe():201] Collecting system info -2023-10-23 09:10:31,368 INFO SystemMonitor:54597 [interfaces.py:start():190] Started cpu monitoring -2023-10-23 09:10:31,368 INFO SystemMonitor:54597 [interfaces.py:start():190] Started disk monitoring -2023-10-23 09:10:31,368 INFO SystemMonitor:54597 [interfaces.py:start():190] Started gpu monitoring -2023-10-23 09:10:31,369 INFO SystemMonitor:54597 [interfaces.py:start():190] Started memory monitoring -2023-10-23 09:10:31,369 INFO SystemMonitor:54597 [interfaces.py:start():190] Started network monitoring -2023-10-23 09:10:31,384 DEBUG HandlerThread:54597 [system_info.py:probe():195] Probing system -2023-10-23 09:10:31,390 DEBUG HandlerThread:54597 [system_info.py:_probe_git():180] Probing git -2023-10-23 09:10:31,403 DEBUG HandlerThread:54597 [system_info.py:_probe_git():188] Probing git done -2023-10-23 09:10:31,403 DEBUG HandlerThread:54597 [system_info.py:probe():240] Probing system done -2023-10-23 09:10:31,403 DEBUG HandlerThread:54597 [system_monitor.py:probe():210] {'os': 'Linux-3.10.0-1160.76.1.el7.x86_64-x86_64-with-glibc2.17', 'python': '3.9.16', 'heartbeatAt': '2023-10-23T06:10:31.384548', 'startedAt': '2023-10-23T06:10:29.992874', 'docker': None, 'cuda': None, 'args': ('--job_name', 'minigpt4_v2_test', '--cfg-path', 'train_configs/minigpt_v2_finetune.yaml'), 'state': 'running', 'program': '/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/train.py', 'codePath': 'train.py', 'git': {'remote': 'https://github.com/junchen14/MiniGPT-4_finetune', 'commit': '5c3ec8bb730b2e9df23b4bb3eb24108f9c04a8d6'}, 'email': 'jun.chen@kaust.edu.sa', 'root': '/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune', 'host': 'gpu201-16-l', 'username': 'chenj0g', 'executable': '/home/chenj0g/anaconda3/envs/eye/bin/python', 'cpu_count': 64, 'cpu_count_logical': 64, 'cpu_freq': {'current': 1996.2770000000016, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}], 'disk': {'total': 251.85773849487305, 'used': 7.905696868896484}, 'gpu': 'NVIDIA A100-SXM4-80GB', 'gpu_count': 2, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}], 'memory': {'total': 503.7154769897461}} -2023-10-23 09:10:31,403 INFO HandlerThread:54597 [system_monitor.py:probe():211] Finished collecting system info -2023-10-23 09:10:31,403 INFO HandlerThread:54597 [system_monitor.py:probe():214] Publishing system info -2023-10-23 09:10:31,403 DEBUG HandlerThread:54597 [system_info.py:_save_pip():51] Saving list of pip packages installed into the current environment -2023-10-23 09:10:31,404 DEBUG HandlerThread:54597 [system_info.py:_save_pip():67] Saving pip packages done -2023-10-23 09:10:31,404 DEBUG HandlerThread:54597 [system_info.py:_save_conda():74] Saving list of conda packages installed into the current environment -2023-10-23 09:10:32,053 INFO Thread-12 :54597 [dir_watcher.py:_on_file_created():278] file/dir created: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:10:32,054 INFO Thread-12 :54597 [dir_watcher.py:_on_file_created():278] file/dir created: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/requirements.txt -2023-10-23 09:10:32,054 INFO Thread-12 :54597 [dir_watcher.py:_on_file_created():278] file/dir created: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/conda-environment.yaml -2023-10-23 09:10:37,055 DEBUG HandlerThread:54597 [system_info.py:_save_conda():86] Saving conda packages done -2023-10-23 09:10:37,057 INFO HandlerThread:54597 [system_monitor.py:probe():216] Finished publishing system info -2023-10-23 09:10:37,057 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/conda-environment.yaml -2023-10-23 09:10:37,058 INFO Thread-12 :54597 [dir_watcher.py:_on_file_created():278] file/dir created: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-metadata.json -2023-10-23 09:10:37,059 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:10:37,060 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: keepalive -2023-10-23 09:10:37,060 DEBUG SenderThread:54597 [sender.py:send():375] send: files -2023-10-23 09:10:37,060 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-metadata.json with policy now -2023-10-23 09:10:37,065 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 09:10:37,066 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: stop_status -2023-10-23 09:10:37,429 DEBUG SenderThread:54597 [sender.py:send():375] send: telemetry -2023-10-23 09:10:37,430 DEBUG SenderThread:54597 [sender.py:send():375] send: telemetry -2023-10-23 09:10:37,867 INFO wandb-upload_0:54597 [upload_job.py:push():137] Uploaded file /tmp/tmpi8sahgaawandb/npzi5ika-wandb-metadata.json -2023-10-23 09:10:40,430 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:10:42,060 INFO Thread-12 :54597 [dir_watcher.py:_on_file_created():278] file/dir created: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/output.log -2023-10-23 09:10:42,566 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:42,567 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:42,567 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:42,569 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:43,061 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:10:43,079 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:43,080 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:43,080 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:43,081 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:43,680 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:43,681 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:43,681 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:43,682 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:44,062 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/output.log -2023-10-23 09:10:44,062 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:10:44,148 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:44,148 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:44,149 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:44,149 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:44,651 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:44,652 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:44,652 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:44,654 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:45,063 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:10:45,120 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:45,121 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:45,121 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:45,122 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:45,631 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:45,632 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:45,632 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:45,632 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:10:45,633 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:46,064 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:10:46,151 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:46,152 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:46,152 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:46,153 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:46,700 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:46,701 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:46,701 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:46,703 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:47,064 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:10:47,182 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:47,183 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:47,183 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:47,184 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:47,656 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:47,657 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:47,657 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:47,658 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:48,065 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:10:48,206 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:48,207 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:48,207 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:48,207 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:48,676 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:48,676 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:48,677 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:48,677 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:49,066 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:10:49,129 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:49,130 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:49,130 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:49,132 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:49,579 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:49,580 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:49,580 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:49,581 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:50,067 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:10:50,137 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:50,138 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:50,138 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:50,139 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:50,686 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:50,687 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:50,687 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:50,687 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:10:50,688 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:51,069 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:10:51,112 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:51,112 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:51,112 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:51,113 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:51,581 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:51,581 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:51,581 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:51,583 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:52,069 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:10:52,088 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 09:10:52,089 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: stop_status -2023-10-23 09:10:52,093 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:52,329 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:52,330 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:52,330 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:52,532 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:52,532 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:52,532 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:52,533 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:53,007 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:53,007 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:53,007 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:53,008 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:53,070 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:10:53,555 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:53,556 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:53,556 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:53,557 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:54,001 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:54,001 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:54,001 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:54,003 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:54,070 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:10:54,510 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:54,511 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:54,511 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:54,512 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:55,015 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:55,015 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:55,016 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:55,016 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:55,071 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:10:55,457 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:55,458 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:55,458 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:55,459 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:55,937 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:55,938 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:55,938 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:55,938 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:10:55,939 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:56,072 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:10:56,406 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:56,407 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:56,407 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:56,409 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:56,859 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:56,860 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:56,860 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:56,861 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:57,073 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:10:57,356 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:57,357 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:57,357 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:57,358 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:57,910 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:57,910 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:57,910 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:57,911 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:58,073 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:10:58,390 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:58,391 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:58,391 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:58,392 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:58,848 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:58,848 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:58,849 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:58,850 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:59,075 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:10:59,420 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:59,420 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:59,421 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:59,421 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:59,871 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:59,871 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:59,872 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:59,873 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:00,075 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:00,395 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:00,396 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:00,396 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:00,397 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:00,900 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:00,900 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:00,901 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:00,902 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:01,076 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:01,379 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:01,379 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:01,383 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:11:01,621 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:01,622 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:01,838 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:01,838 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:01,838 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:01,839 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:02,076 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:02,077 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/config.yaml -2023-10-23 09:11:02,317 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:02,318 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:02,318 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:02,319 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:02,825 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:02,826 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:02,826 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:02,827 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:03,077 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:03,270 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:03,271 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:03,271 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:03,273 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:03,742 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:03,742 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:03,743 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:03,743 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:04,078 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:04,285 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:04,286 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:04,286 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:04,287 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:04,767 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:04,767 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:04,767 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:04,768 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:05,079 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:05,227 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:05,228 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:05,228 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:05,229 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:05,692 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:05,693 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:05,693 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:05,695 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:06,079 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:06,114 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:06,114 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:06,115 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:06,115 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:06,732 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:06,733 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:06,733 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:06,733 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:11:06,734 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:07,080 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:07,088 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 09:11:07,088 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: stop_status -2023-10-23 09:11:07,282 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:07,372 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:07,372 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:07,373 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:07,775 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:07,776 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:07,776 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:07,778 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:08,081 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:08,257 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:08,257 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:08,258 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:08,258 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:08,886 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:08,887 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:08,887 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:08,888 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:09,082 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:09,348 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:09,348 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:09,349 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:09,349 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:09,850 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:09,851 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:09,851 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:09,853 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:10,082 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:10,319 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:10,320 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:10,320 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:10,321 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:10,818 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:10,819 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:10,819 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:10,820 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:11,083 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:11,292 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:11,293 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:11,293 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:11,294 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:11,762 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:11,762 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:11,763 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:11,763 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:11:11,763 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:12,084 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:12,244 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:12,244 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:12,244 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:12,246 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:12,727 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:12,728 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:12,728 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:12,729 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:13,085 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:13,204 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:13,205 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:13,205 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:13,209 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:13,762 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:13,763 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:13,763 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:13,764 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:14,085 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:14,249 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:14,250 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:14,250 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:14,252 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:14,757 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:14,758 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:14,758 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:14,759 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:15,086 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:15,207 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:15,208 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:15,208 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:15,208 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:15,653 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:15,654 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:15,654 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:15,655 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:16,087 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:16,163 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:16,164 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:16,164 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:16,165 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:16,632 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:16,632 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:16,633 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:16,634 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:17,088 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:17,105 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:17,105 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:17,106 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:17,106 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:11:17,106 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:17,650 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:17,651 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:17,651 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:17,651 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:18,090 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:18,137 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:18,138 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:18,138 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:18,139 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:18,650 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:18,650 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:18,651 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:18,653 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:19,092 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:19,127 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:19,128 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:19,128 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:19,129 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:19,578 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:19,579 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:19,579 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:19,580 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:20,093 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:20,176 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:20,177 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:20,177 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:20,177 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:20,608 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:20,608 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:20,608 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:20,612 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:21,093 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:21,169 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:21,169 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:21,170 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:21,171 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:21,748 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:21,748 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:21,749 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:21,749 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:22,088 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 09:11:22,088 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: stop_status -2023-10-23 09:11:22,094 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:22,298 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:22,356 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:22,357 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:11:22,357 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:22,358 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:22,769 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:22,770 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:22,770 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:22,771 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:23,095 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:23,289 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:23,289 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:23,289 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:23,291 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:23,846 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:23,846 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:23,847 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:23,848 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:24,096 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:24,310 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:24,311 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:24,311 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:24,312 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:24,832 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:24,833 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:24,833 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:24,834 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:25,096 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:25,284 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:25,285 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:25,285 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:25,286 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:25,790 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:25,790 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:25,790 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:25,792 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:26,097 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:26,279 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:26,280 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:26,280 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:26,281 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:26,747 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:26,748 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:26,748 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:26,748 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:27,098 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:27,260 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:27,261 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:27,261 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:27,262 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:27,733 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:27,733 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:27,734 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:27,734 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:11:27,734 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:28,099 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:28,180 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:28,181 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:28,181 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:28,183 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:28,641 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:28,642 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:28,642 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:28,643 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:29,028 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:29,029 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:29,029 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:29,030 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:29,099 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:29,495 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:29,495 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:29,496 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:29,496 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:30,045 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:30,046 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:30,046 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:30,047 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:30,100 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:30,508 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:30,509 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:30,509 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:30,510 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:30,990 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:30,991 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:30,991 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:30,991 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:31,101 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:31,369 DEBUG SystemMonitor:54597 [system_monitor.py:_start():159] Starting system metrics aggregation loop -2023-10-23 09:11:31,371 DEBUG SenderThread:54597 [sender.py:send():375] send: stats -2023-10-23 09:11:31,540 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:31,541 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:31,541 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:31,542 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:32,060 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:32,061 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:32,061 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:32,062 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:32,102 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:32,502 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:32,502 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:32,503 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:32,503 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:32,997 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:32,998 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:32,998 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:32,998 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:11:32,999 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:33,102 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:33,467 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:33,468 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:33,468 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:33,469 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:33,994 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:33,995 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:33,995 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:33,996 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:34,103 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:34,464 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:34,465 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:34,465 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:34,466 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:35,003 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:35,004 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:35,004 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:35,005 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:35,104 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:35,469 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:35,470 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:35,470 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:35,471 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:35,929 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:35,930 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:35,930 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:35,931 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:36,105 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:36,383 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:36,383 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:36,384 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:36,384 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:36,899 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:36,899 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:36,899 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:36,900 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:37,088 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 09:11:37,088 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: stop_status -2023-10-23 09:11:37,105 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:37,495 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:37,496 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:37,496 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:37,497 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:37,873 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:37,874 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:37,874 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:37,875 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:38,107 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:38,402 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:38,403 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:38,403 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:38,404 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:11:38,405 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:38,928 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:38,929 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:38,929 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:38,930 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:39,109 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:39,516 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:39,516 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:39,517 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:39,517 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:39,995 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:39,996 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:39,996 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:39,996 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:40,110 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:40,447 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:40,448 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:40,448 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:40,449 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:41,075 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:41,076 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:41,076 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:41,077 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:41,111 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:41,528 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:41,529 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:41,529 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:41,530 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:42,011 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:42,012 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:42,012 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:42,013 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:42,112 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:42,564 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:42,565 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:42,565 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:42,566 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:43,031 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:43,032 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:43,032 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:43,033 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:43,112 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:43,488 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:43,489 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:43,489 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:43,489 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:11:43,490 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:43,935 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:43,935 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:43,935 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:43,936 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:44,113 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:44,437 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:44,437 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:44,438 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:44,438 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:44,888 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:44,889 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:44,889 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:44,890 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:45,114 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:45,355 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:45,356 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:45,356 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:45,356 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:45,816 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:45,816 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:45,817 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:45,817 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:46,115 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:46,301 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:46,301 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:46,302 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:46,302 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:46,773 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:46,774 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:46,774 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:46,775 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:47,116 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:47,298 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:47,299 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:47,299 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:47,300 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:47,772 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:47,772 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:47,773 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:47,773 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:48,116 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:48,217 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:48,218 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:48,218 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:48,219 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:48,690 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:48,691 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:48,691 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:48,692 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:11:48,693 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:49,117 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:49,135 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:49,136 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:49,136 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:49,136 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:49,674 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:49,675 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:49,675 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:49,676 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:50,118 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:50,119 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:50,119 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:50,119 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:50,120 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:50,630 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:50,630 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:50,631 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:50,631 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:51,119 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:51,155 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:51,156 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:51,156 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:51,157 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:51,623 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:51,624 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:51,624 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:51,625 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:52,054 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:52,054 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:52,055 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:52,055 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:52,088 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 09:11:52,088 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: stop_status -2023-10-23 09:11:52,121 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:52,598 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:52,599 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:52,599 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:52,600 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:53,122 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:53,166 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:53,166 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:53,166 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:53,167 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:53,687 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:53,687 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:53,688 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:53,688 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:54,122 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:54,143 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:54,144 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:54,144 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:54,144 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:11:54,145 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:54,601 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:54,601 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:54,602 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:54,602 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:55,031 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:55,032 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:55,032 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:55,034 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:55,123 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:55,591 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:55,592 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:55,592 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:55,593 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:56,049 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:56,049 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:56,049 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:56,050 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:56,124 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:56,553 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:56,554 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:56,554 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:56,555 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:57,051 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:57,052 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:57,052 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:57,053 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:57,125 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:57,478 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:57,479 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:57,479 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:57,479 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:57,909 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:57,910 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:57,910 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:57,911 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:58,125 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:58,318 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:58,318 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:58,318 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:58,319 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:58,868 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:58,869 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:58,869 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:58,871 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:59,126 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:59,339 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:59,340 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:59,340 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:59,340 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:11:59,341 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:59,788 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:59,788 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:59,789 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:59,789 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:00,127 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:00,237 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:00,238 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:00,238 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:00,239 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:00,687 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:00,687 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:00,687 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:00,688 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:01,128 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:01,219 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:01,219 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:01,219 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:01,220 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:01,371 DEBUG SenderThread:54597 [sender.py:send():375] send: stats -2023-10-23 09:12:01,667 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:01,668 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:01,668 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:01,669 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:02,129 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:02,197 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:02,198 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:02,198 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:02,199 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:02,726 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:02,727 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:02,727 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:02,728 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:03,129 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:03,154 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:03,154 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:03,154 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:03,155 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:03,624 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:03,625 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:03,625 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:03,626 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:04,130 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:04,155 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:04,155 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:04,155 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:04,156 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:04,742 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:04,743 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:04,743 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:04,743 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:12:04,744 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:05,131 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:05,196 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:05,197 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:05,197 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:05,198 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:05,680 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:05,681 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:05,681 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:05,682 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:06,132 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:06,166 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:06,167 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:06,167 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:06,168 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:06,674 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:06,675 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:06,675 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:06,676 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:07,119 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 09:12:07,120 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: stop_status -2023-10-23 09:12:07,123 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:07,132 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:07,339 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:07,339 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:07,340 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:07,594 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:07,595 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:07,595 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:07,595 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:08,061 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:08,061 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:08,062 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:08,063 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:08,133 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:08,489 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:08,490 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:08,490 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:08,491 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:08,940 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:08,941 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:08,941 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:08,943 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:09,134 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:09,413 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:09,414 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:09,414 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:09,415 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:09,894 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:09,895 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:09,895 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:09,895 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:12:09,896 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:10,135 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:10,372 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:10,372 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:10,372 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:10,373 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:10,880 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:10,880 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:10,880 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:10,881 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:11,135 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:11,394 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:11,395 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:11,395 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:11,396 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:11,949 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:11,950 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:11,950 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:11,951 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:12,136 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:12,471 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:12,472 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:12,472 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:12,473 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:12,992 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:12,993 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:12,993 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:12,994 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:13,137 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:13,524 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:13,525 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:13,525 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:13,526 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:14,029 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:14,030 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:14,030 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:14,031 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:14,138 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:14,592 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:14,592 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:14,593 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:14,593 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:15,091 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:15,092 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:15,092 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:15,092 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:12:15,093 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:15,139 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:15,639 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:15,640 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:15,640 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:15,641 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:16,118 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:16,119 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:16,119 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:16,120 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:16,139 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:16,546 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:16,547 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:16,547 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:16,548 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:17,105 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:17,106 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:17,106 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:17,107 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:17,140 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:17,577 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:17,577 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:17,577 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:17,578 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:18,141 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:18,170 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:18,171 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:18,171 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:18,172 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:18,667 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:18,667 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:18,668 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:18,668 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:19,094 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:19,094 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:19,094 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:19,097 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:19,142 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:19,692 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:19,693 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:19,693 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:19,694 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:20,142 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:20,228 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:20,229 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:20,229 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:20,229 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:12:20,230 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:20,672 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:20,672 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:20,672 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:20,673 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:21,143 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:21,156 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:21,157 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:21,157 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:21,158 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:21,718 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:21,718 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:21,718 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:21,719 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:22,119 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 09:12:22,120 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: stop_status -2023-10-23 09:12:22,144 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:22,186 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:22,415 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:22,416 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:22,416 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:22,786 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:22,787 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:22,787 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:22,788 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:23,145 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:23,324 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:23,325 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:23,325 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:23,325 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:23,809 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:23,809 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:23,810 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:23,810 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:24,145 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:24,276 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:24,277 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:24,277 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:24,278 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:24,797 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:24,797 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:24,797 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:24,798 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:25,146 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:25,336 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:25,336 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:25,337 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:25,337 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:12:25,337 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:25,813 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:25,813 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:25,813 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:25,814 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:26,147 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:26,328 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:26,329 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:26,329 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:26,330 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:26,770 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:26,771 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:26,771 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:26,772 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:27,147 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:27,263 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:27,264 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:27,264 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:27,265 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:27,653 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:27,653 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:27,654 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:27,654 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:28,148 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:28,219 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:28,219 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:28,220 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:28,220 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:28,685 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:28,686 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:28,686 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:28,687 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:29,145 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:29,146 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:29,146 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:29,148 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:29,149 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:29,703 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:29,704 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:29,704 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:29,705 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:30,149 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:30,184 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:30,184 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:30,184 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:30,185 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:30,729 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:30,730 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:30,730 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:30,730 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:12:30,731 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:31,150 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:31,236 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:31,237 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:31,237 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:31,237 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:31,372 DEBUG SenderThread:54597 [sender.py:send():375] send: stats -2023-10-23 09:12:31,738 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:31,738 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:31,738 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:31,739 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:32,151 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:32,248 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:32,248 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:32,248 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:32,249 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:32,654 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:32,655 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:32,655 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:32,656 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:33,108 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:33,108 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:33,109 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:33,110 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:33,152 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:33,666 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:33,667 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:33,667 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:33,668 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:34,152 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:34,210 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:34,211 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:34,211 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:34,211 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:34,654 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:34,654 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:34,654 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:34,655 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:35,101 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:35,101 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:35,101 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:35,102 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:35,153 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:35,651 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:35,651 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:35,651 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:35,652 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:36,098 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:36,098 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:36,098 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:36,098 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:12:36,099 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:36,154 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:36,644 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:36,645 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:36,645 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:36,646 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:37,150 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 09:12:37,150 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: stop_status -2023-10-23 09:12:37,153 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:37,155 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:37,409 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:37,409 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:37,410 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:37,582 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:37,583 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:37,583 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:37,584 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:38,155 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:38,180 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:38,180 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:38,180 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:38,181 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:38,727 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:38,728 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:38,728 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:38,728 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:39,156 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:39,226 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:39,227 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:39,227 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:39,229 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:39,735 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:39,736 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:39,736 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:39,737 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:40,157 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:40,215 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:40,216 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:40,216 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:40,217 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:40,690 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:40,690 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:40,690 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:40,691 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:41,158 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:41,260 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:41,261 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:41,261 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:41,261 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:12:41,262 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:41,715 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:41,715 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:41,715 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:41,716 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:42,158 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:42,285 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:42,286 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:42,286 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:42,287 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:42,766 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:42,767 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:42,767 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:42,768 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:43,159 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:43,212 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:43,213 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:43,213 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:43,214 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:43,666 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:43,666 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:43,666 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:43,667 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:44,163 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:44,226 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:44,227 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:44,227 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:44,228 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:44,732 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:44,732 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:44,732 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:44,733 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:45,159 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:45,159 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:45,160 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:45,161 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:45,163 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:45,663 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:45,664 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:45,664 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:45,665 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:46,115 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:46,115 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:46,116 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:46,116 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:46,164 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:46,562 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:46,563 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:46,563 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:46,563 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:12:46,564 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:47,112 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:47,113 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:47,113 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:47,116 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:47,165 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:47,557 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:47,557 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:47,557 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:47,558 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:48,121 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:48,122 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:48,122 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:48,123 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:48,166 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:48,601 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:48,601 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:48,601 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:48,602 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:49,082 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:49,083 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:49,083 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:49,084 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:49,167 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:49,646 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:49,646 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:49,646 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:49,648 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:50,167 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:50,179 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:50,179 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:50,180 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:50,180 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:50,597 DEBUG SenderThread:54597 [sender.py:send():375] send: exit -2023-10-23 09:12:50,597 INFO SenderThread:54597 [sender.py:send_exit():598] handling exit code: 255 -2023-10-23 09:12:50,597 INFO SenderThread:54597 [sender.py:send_exit():600] handling runtime: 139 -2023-10-23 09:12:50,598 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:50,598 INFO SenderThread:54597 [sender.py:send_exit():606] send defer -2023-10-23 09:12:50,598 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:12:50,598 INFO HandlerThread:54597 [handler.py:handle_request_defer():170] handle defer: 0 -2023-10-23 09:12:50,598 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: defer -2023-10-23 09:12:50,598 INFO SenderThread:54597 [sender.py:send_request_defer():622] handle sender defer: 0 -2023-10-23 09:12:50,598 INFO SenderThread:54597 [sender.py:transition_state():626] send defer: 1 -2023-10-23 09:12:50,598 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:12:50,599 INFO HandlerThread:54597 [handler.py:handle_request_defer():170] handle defer: 1 -2023-10-23 09:12:50,599 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: defer -2023-10-23 09:12:50,599 INFO SenderThread:54597 [sender.py:send_request_defer():622] handle sender defer: 1 -2023-10-23 09:12:50,599 INFO SenderThread:54597 [sender.py:transition_state():626] send defer: 2 -2023-10-23 09:12:50,599 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:12:50,599 INFO HandlerThread:54597 [handler.py:handle_request_defer():170] handle defer: 2 -2023-10-23 09:12:50,599 INFO HandlerThread:54597 [system_monitor.py:finish():190] Stopping system monitor -2023-10-23 09:12:50,599 DEBUG SystemMonitor:54597 [system_monitor.py:_start():166] Finished system metrics aggregation loop -2023-10-23 09:12:50,600 INFO HandlerThread:54597 [interfaces.py:finish():202] Joined cpu monitor -2023-10-23 09:12:50,600 DEBUG SystemMonitor:54597 [system_monitor.py:_start():170] Publishing last batch of metrics -2023-10-23 09:12:50,600 INFO HandlerThread:54597 [interfaces.py:finish():202] Joined disk monitor -2023-10-23 09:12:50,702 INFO HandlerThread:54597 [interfaces.py:finish():202] Joined gpu monitor -2023-10-23 09:12:50,703 INFO HandlerThread:54597 [interfaces.py:finish():202] Joined memory monitor -2023-10-23 09:12:50,703 INFO HandlerThread:54597 [interfaces.py:finish():202] Joined network monitor -2023-10-23 09:12:50,703 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: defer -2023-10-23 09:12:50,703 INFO SenderThread:54597 [sender.py:send_request_defer():622] handle sender defer: 2 -2023-10-23 09:12:50,703 INFO SenderThread:54597 [sender.py:transition_state():626] send defer: 3 -2023-10-23 09:12:50,703 DEBUG SenderThread:54597 [sender.py:send():375] send: stats -2023-10-23 09:12:50,703 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:12:50,704 INFO HandlerThread:54597 [handler.py:handle_request_defer():170] handle defer: 3 -2023-10-23 09:12:50,704 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: defer -2023-10-23 09:12:50,704 INFO SenderThread:54597 [sender.py:send_request_defer():622] handle sender defer: 3 -2023-10-23 09:12:50,704 INFO SenderThread:54597 [sender.py:transition_state():626] send defer: 4 -2023-10-23 09:12:50,704 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:12:50,704 INFO HandlerThread:54597 [handler.py:handle_request_defer():170] handle defer: 4 -2023-10-23 09:12:50,704 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: defer -2023-10-23 09:12:50,704 INFO SenderThread:54597 [sender.py:send_request_defer():622] handle sender defer: 4 -2023-10-23 09:12:50,705 INFO SenderThread:54597 [sender.py:transition_state():626] send defer: 5 -2023-10-23 09:12:50,705 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:12:50,705 INFO HandlerThread:54597 [handler.py:handle_request_defer():170] handle defer: 5 -2023-10-23 09:12:50,705 DEBUG SenderThread:54597 [sender.py:send():375] send: summary -2023-10-23 09:12:50,706 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:50,706 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: defer -2023-10-23 09:12:50,706 INFO SenderThread:54597 [sender.py:send_request_defer():622] handle sender defer: 5 -2023-10-23 09:12:50,706 INFO SenderThread:54597 [sender.py:transition_state():626] send defer: 6 -2023-10-23 09:12:50,706 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:12:50,706 INFO HandlerThread:54597 [handler.py:handle_request_defer():170] handle defer: 6 -2023-10-23 09:12:50,706 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: defer -2023-10-23 09:12:50,706 INFO SenderThread:54597 [sender.py:send_request_defer():622] handle sender defer: 6 -2023-10-23 09:12:50,706 INFO SenderThread:54597 [sender.py:transition_state():626] send defer: 7 -2023-10-23 09:12:50,706 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:12:50,706 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:12:50,706 INFO HandlerThread:54597 [handler.py:handle_request_defer():170] handle defer: 7 -2023-10-23 09:12:50,707 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: defer -2023-10-23 09:12:50,707 INFO SenderThread:54597 [sender.py:send_request_defer():622] handle sender defer: 7 -2023-10-23 09:12:51,168 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:51,597 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: poll_exit -2023-10-23 09:12:51,952 INFO SenderThread:54597 [sender.py:transition_state():626] send defer: 8 -2023-10-23 09:12:51,952 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: poll_exit -2023-10-23 09:12:51,952 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:12:51,952 INFO HandlerThread:54597 [handler.py:handle_request_defer():170] handle defer: 8 -2023-10-23 09:12:51,952 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: defer -2023-10-23 09:12:51,952 INFO SenderThread:54597 [sender.py:send_request_defer():622] handle sender defer: 8 -2023-10-23 09:12:51,962 INFO SenderThread:54597 [sender.py:transition_state():626] send defer: 9 -2023-10-23 09:12:51,962 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:12:51,962 DEBUG SenderThread:54597 [sender.py:send():375] send: artifact -2023-10-23 09:12:51,962 INFO HandlerThread:54597 [handler.py:handle_request_defer():170] handle defer: 9 -2023-10-23 09:12:52,169 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/output.log -2023-10-23 09:12:52,598 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: poll_exit -2023-10-23 09:12:53,080 INFO SenderThread:54597 [sender.py:send_artifact():1474] sent artifact job-https___github.com_junchen14_MiniGPT-4_finetune_train.py - {'id': 'QXJ0aWZhY3Q6NjEzMjczODU5', 'digest': 'e1357229d7190882bf8b2dba86a37a56', 'state': 'COMMITTED', 'aliases': [{'artifactCollectionName': 'job-https___github.com_junchen14_MiniGPT-4_finetune_train.py', 'alias': 'latest'}, {'artifactCollectionName': 'job-https___github.com_junchen14_MiniGPT-4_finetune_train.py', 'alias': 'v0'}], 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjEwOTExOTI4MQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6NjEzMjczODU5', 'versionIndex': 0}}, 'version': 'v0'} -2023-10-23 09:12:53,080 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: defer -2023-10-23 09:12:53,080 INFO SenderThread:54597 [sender.py:send_request_defer():622] handle sender defer: 9 -2023-10-23 09:12:53,080 INFO SenderThread:54597 [dir_watcher.py:finish():365] shutting down directory watcher -2023-10-23 09:12:53,170 INFO SenderThread:54597 [dir_watcher.py:finish():395] scan: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files -2023-10-23 09:12:53,170 INFO SenderThread:54597 [dir_watcher.py:finish():409] scan save: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/config.yaml config.yaml -2023-10-23 09:12:53,170 INFO SenderThread:54597 [dir_watcher.py:finish():409] scan save: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/output.log output.log -2023-10-23 09:12:53,173 INFO SenderThread:54597 [dir_watcher.py:finish():409] scan save: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-metadata.json wandb-metadata.json -2023-10-23 09:12:53,173 INFO SenderThread:54597 [dir_watcher.py:finish():409] scan save: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/conda-environment.yaml conda-environment.yaml -2023-10-23 09:12:53,176 INFO SenderThread:54597 [dir_watcher.py:finish():409] scan save: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/requirements.txt requirements.txt -2023-10-23 09:12:53,178 INFO SenderThread:54597 [dir_watcher.py:finish():409] scan save: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json wandb-summary.json -2023-10-23 09:12:53,178 INFO SenderThread:54597 [sender.py:transition_state():626] send defer: 10 -2023-10-23 09:12:53,178 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: poll_exit -2023-10-23 09:12:53,180 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:12:53,183 INFO HandlerThread:54597 [handler.py:handle_request_defer():170] handle defer: 10 -2023-10-23 09:12:53,183 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: defer -2023-10-23 09:12:53,183 INFO SenderThread:54597 [sender.py:send_request_defer():622] handle sender defer: 10 -2023-10-23 09:12:53,183 INFO SenderThread:54597 [file_pusher.py:finish():168] shutting down file pusher -2023-10-23 09:12:53,598 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: poll_exit -2023-10-23 09:12:53,599 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: poll_exit -2023-10-23 09:12:53,859 INFO wandb-upload_2:54597 [upload_job.py:push():137] Uploaded file /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/conda-environment.yaml -2023-10-23 09:12:54,033 INFO wandb-upload_0:54597 [upload_job.py:push():137] Uploaded file /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/config.yaml -2023-10-23 09:12:54,177 INFO wandb-upload_1:54597 [upload_job.py:push():137] Uploaded file /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/output.log -2023-10-23 09:12:54,187 INFO wandb-upload_4:54597 [upload_job.py:push():137] Uploaded file /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:54,206 INFO wandb-upload_3:54597 [upload_job.py:push():137] Uploaded file /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/requirements.txt -2023-10-23 09:12:54,407 INFO Thread-11 :54597 [sender.py:transition_state():626] send defer: 11 -2023-10-23 09:12:54,407 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:12:54,407 INFO HandlerThread:54597 [handler.py:handle_request_defer():170] handle defer: 11 -2023-10-23 09:12:54,407 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: defer -2023-10-23 09:12:54,407 INFO SenderThread:54597 [sender.py:send_request_defer():622] handle sender defer: 11 -2023-10-23 09:12:54,407 INFO SenderThread:54597 [file_pusher.py:join():173] waiting for file pusher -2023-10-23 09:12:54,407 INFO SenderThread:54597 [sender.py:transition_state():626] send defer: 12 -2023-10-23 09:12:54,408 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:12:54,408 INFO HandlerThread:54597 [handler.py:handle_request_defer():170] handle defer: 12 -2023-10-23 09:12:54,408 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: defer -2023-10-23 09:12:54,408 INFO SenderThread:54597 [sender.py:send_request_defer():622] handle sender defer: 12 -2023-10-23 09:12:54,599 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: poll_exit -2023-10-23 09:12:55,181 WARNING StreamThr :54597 [internal.py:is_dead():414] Internal process exiting, parent pid 53640 disappeared -2023-10-23 09:12:55,181 ERROR StreamThr :54597 [internal.py:wandb_internal():152] Internal process shutdown. -2023-10-23 09:12:55,599 INFO HandlerThread:54597 [handler.py:finish():845] shutting down handler -2023-10-23 09:12:55,599 INFO WriterThread:54597 [datastore.py:close():298] close: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/run-txn4i28h.wandb -2023-10-23 09:12:56,034 INFO SenderThread:54597 [sender.py:transition_state():626] send defer: 13 -2023-10-23 09:12:56,034 INFO SenderThread:54597 [sender.py:finish():1550] shutting down sender -2023-10-23 09:12:56,034 INFO SenderThread:54597 [file_pusher.py:finish():168] shutting down file pusher -2023-10-23 09:12:56,034 INFO SenderThread:54597 [file_pusher.py:join():173] waiting for file pusher diff --git a/wandb/run-20231023_091029-txn4i28h/logs/debug.log b/wandb/run-20231023_091029-txn4i28h/logs/debug.log deleted file mode 100644 index a67e3c3..0000000 --- a/wandb/run-20231023_091029-txn4i28h/logs/debug.log +++ /dev/null @@ -1,26 +0,0 @@ -2023-10-23 09:10:30,028 INFO MainThread:53640 [wandb_setup.py:_flush():76] Configure stats pid to 53640 -2023-10-23 09:10:30,028 INFO MainThread:53640 [wandb_setup.py:_flush():76] Loading settings from /home/chenj0g/.config/wandb/settings -2023-10-23 09:10:30,028 INFO MainThread:53640 [wandb_setup.py:_flush():76] Loading settings from /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/settings -2023-10-23 09:10:30,028 INFO MainThread:53640 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} -2023-10-23 09:10:30,028 INFO MainThread:53640 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'train.py', 'program': '/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/train.py'} -2023-10-23 09:10:30,028 INFO MainThread:53640 [wandb_init.py:_log_setup():507] Logging user logs to /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/logs/debug.log -2023-10-23 09:10:30,028 INFO MainThread:53640 [wandb_init.py:_log_setup():508] Logging internal logs to /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/logs/debug-internal.log -2023-10-23 09:10:30,028 INFO MainThread:53640 [wandb_init.py:init():547] calling init triggers -2023-10-23 09:10:30,029 INFO MainThread:53640 [wandb_init.py:init():554] wandb.init called with sweep_config: {} -config: {} -2023-10-23 09:10:30,029 INFO MainThread:53640 [wandb_init.py:init():595] starting backend -2023-10-23 09:10:30,029 INFO MainThread:53640 [wandb_init.py:init():599] setting up manager -2023-10-23 09:10:30,030 INFO MainThread:53640 [backend.py:_multiprocessing_setup():106] multiprocessing start_methods=fork,spawn,forkserver, using: spawn -2023-10-23 09:10:30,035 INFO MainThread:53640 [wandb_init.py:init():605] backend started and connected -2023-10-23 09:10:30,037 INFO MainThread:53640 [wandb_init.py:init():695] updated telemetry -2023-10-23 09:10:30,446 INFO MainThread:53640 [wandb_init.py:init():732] communicating run to backend with 60.0 second timeout -2023-10-23 09:10:31,053 INFO MainThread:53640 [wandb_run.py:_on_init():2176] communicating current version -2023-10-23 09:10:31,357 INFO MainThread:53640 [wandb_run.py:_on_init():2185] got version response upgrade_message: "wandb version 0.15.12 is available! To upgrade, please run:\n $ pip install wandb --upgrade" - -2023-10-23 09:10:31,357 INFO MainThread:53640 [wandb_init.py:init():782] starting run threads in backend -2023-10-23 09:10:37,065 INFO MainThread:53640 [wandb_run.py:_console_start():2157] atexit reg -2023-10-23 09:10:37,065 INFO MainThread:53640 [wandb_run.py:_redirect():2012] redirect: SettingsConsole.WRAP_RAW -2023-10-23 09:10:37,065 INFO MainThread:53640 [wandb_run.py:_redirect():2077] Wrapping output streams. -2023-10-23 09:10:37,065 INFO MainThread:53640 [wandb_run.py:_redirect():2102] Redirects installed. -2023-10-23 09:10:37,066 INFO MainThread:53640 [wandb_init.py:init():824] run started, returning control to user process -2023-10-23 09:10:37,066 INFO MainThread:53640 [wandb_watch.py:watch():51] Watching diff --git a/wandb/run-20231023_091029-txn4i28h/run-txn4i28h.wandb b/wandb/run-20231023_091029-txn4i28h/run-txn4i28h.wandb deleted file mode 100644 index 313e061283e6d5bcc54cc055536e28954c04095b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 48616 zcmchg2bg44wYE9k(=arH3^O7yfD9uds8d~a@&OdVOA%r4>-;4Ud}XG7MxgD0;z7fHHKdC8NIWShC(78U;FW%=PDUB?`W!)?9d(vAi5 z=KeA_`vrG?;Ox#tXLs}rkF6f;k*QYHm-qCJ_V$mB4)g{+gQM#vdj>ZS_N*G|>lxoT zH9R(o6ZA}uObsp_@9kgNyL@o6XKHMsf4FDu)X1tyeMJAt@v)K7soMC){e#B-i%*yw zoH(g>VsvD5`TmQK85+^-j-#By@ zw%umIY2Dj(cMXn@^$&0Myn6Sd?m4T*CMP$0JF+<%K|RYFSr8?DmZjZ$b$2hD8d=Rg z>|H&+*;~+vv$P&I!qBfpaTKLN(!FQ*++`DMNA*|w0~UgMy%8n=N!W;!NME*Wa%zyb zd-Vld{q`NTwQ}~k3(hpf*sUqXf<`Ti{J0U-eLs$}px$uE5!C8Q5~qG-%fUkEHyS|{ z`cV`oaZr?_v7lqF_rqiE^fn{OwhPWMC7D%7y_RM5Ea5-(I8CCsQLT`m7AI+xq+wZ* zI7A~^=!bqii!wtYK7w={bLrHVBuKX{NKqwmkk(^AOp->Ukv8J6d$DPn9b2XeK|2I4 z3Bv0QFh0!U7_~-WT+~Q_Af3Da_Ufx7$ac0Mvl?mCf-pgyjVQ??v>R1xB&@|@mSO^1 zBP=weji3?vnIA-HQIHTp=Ja3IbCd*`XA4r)2-d}{bw3Pnks$M{H4^d5(=;vlNK~t* zLC}cnX_EMUk{E(S2-0=o8y+B#8VdjiM-Q`23zQbqNx4+SYwrBT)@MPXj-T zF-{cuNl_y)f^5C=n+tc8AWv}#!cWNjdYFbdeHz9|wIDG*l7&%e_wsrpKvi`=^3x#2 zc7@%WAjme?|K^_WNst|!f@F9Z9}xv{k_541|Hd^CB}js_`{_j-!JBJw?1MFF*vNo` zuqa51Aai&BdDl13Mv(anPS;v{JmcH5I0*67G{PCH6_V5%>@POB1!ue-Ms>8>sPq1M zXjmpgj%`2m!QCE~LUyzjQrt9g7^YExR@ioFBdp#uNiB{-wnc2~gM~(##dZBN@p098 zEm-Yn)Dfh+BVIXQhxdX7XL+VZin}I^vINs40UNLZrd5lQ*6=025yZAmQi7_KUj&ZC zNduraWD&M*SHJ)ML#333E~WU`#m@+w@Yc+QBq`V$HrS}PQdq#Su^;3283%zOh_H3u zyuXjaI+lK{I8pi-%$XObi`&b~m;D8-FU!d)?$hCcNvINX(;CIt;#hzwp zGQSy`M8393DIoB9nd1{Dk!=EgnV}S+?83L*yw#~P&u%XBgoFZVl4%P9wSIL(fFNLO zh98)4k0cz6VLhq`4S^rWPQg6BVD_m${>|+#lq9>mBoPdtl6sIO>_vW1RUjP^r*g7@ zV7ANpHKGAFRw8iJNnDgfAiLA@b*KDAj;psEK#M3k&-pnZ zUmO$k6}2Lu-TBA<-gihVdpflevXMb+pn^!5z`a_Ouofvk%M}Qqv*GS+TDk?KlE#Y+Pz&+8xiDy zksK}HONLr1?Gx9?MoHCdSmAZQfm-W8KX72eilBCnKV5vrA0)^#Z9$6WNfi*2IR*8U z@I9+mNm3&PCs>SZL0FI+6cIvBpV-(vfspp;`+jHW%@SlEmmrM>g$9mfP9NaX;jl?9 zt0Q&EG)WC_MLwNAb+E-ad;$UOo|kmpIwUuFmeWlHNGV^zaVlJJ0Bu?$$BM~(ZI@+1 zXF4Yb=Q6^YvIuAw|6uZtK(g%Xlm)GD@?e`#g$J^$N|#M*WQs^!(hCcD&Vvut*~aYj zg3AVi+P(HW=DcIIk@vGVaxr?tQ5s~Vpa~x5%6*~?=+Le&MQ<7UVh^Vjm2g?o$)Y3z z+9gXn9(}1KdA3UuVgruadXi=&6G&98l&qF=wy?=;r)41sVq)D!1So+yMM(s+&sfxZ z@J}Vlb6k?(Z-71erF@YDVRb|s*J~hKgwU2G0SggndV};@8LDX=!R+4tO;@jzr##o` zDeO)lpU6O2C7)KQ6hz@fuCv!HuSFD`ncYWd7m*DT^cJ-upnc{o2i&x+lJ%v|WF4<( zgZhaNR;Jb17E^=*jq91M6BVbhY+XBRU3WB7lA2c`94Y~{*=e)HJL{m>S;-x?!`>daQ`Uyl)$a1v*{G;qz-at_?|qvCp1=(DqkI65k~Wryr_Su`IyGeo%Q7BT0f^Cw6Tq z;PpD#3(k-TvT+Q54TZ4J*{AD@=gLh2+f9nrAvRS2Boj442iC_l!E{(hHoyifpeP`k zGd;sy43!9ApL@m1`@bcXgf5j}lvptml|v=U4u1`6m@zNwSv!XXU^Uh8N1k8;Y$$+T z`sk;2d`wUh*-%n+6Ou_lG3KZEA@!xG+Bgx7LCS-ny>X}+6L=ydIi+e~1xO18ueCS+ zT2sG zHN268G_n1J1?o9~VnfNK;U|#*)_>W}kG)kY+25rSHPf&%0c5I-$KfVPjhZ$=P)Q{$ zs9;Yl8#NL(b5SLc05N922TN--T)f^*-oLQHg#;#Kx&Aw8g}!)F?});bqF`R3>pW3NQweL_*i( zML#+9L800~Zd400zMtfcG>7A>IwMX9Qwb_Dp@nK;owEo(r4B>xr81 z?Dpd4YLF@tEj}Lr6T6tl$R?vCwBY{uuMIy+7 zUKOw}hL-1rl?b{6M36#zjeDVeEmsKkeF2Ya9QUYa0{E$G`+-bJ9o}*p=HD zNeKKgWLXd;CNL7}>cE-ii3PFGt6lTkw@8pfU4oFjpcQ&wBzAyoqgoI`RqV$;HSr`2 z=)iGe&{7fFa8MU)6ANOWAB~^3lWcQXwQb0j2nbX;;m7D4tghlDgsL$CK~UN|EKrCd zwI@%Z(qYFy276QS(E*FN=Q zJ(A=|rzCh9y@+}p*m9OD5@J~Dft{vo~3u9k+`lD|>Ns=7pl7y~k7^U*ph~PeQ zcq}n2IRGUiTPG~g#Gq2aQ2?+RC7f6o`(J;*=cLaYj6 zP`EEkLU@D2suPj06AkMKVGsT8NeAsHNse|&LQR-B%vWCq#XG7531kL}R%DB|BuPyb zK=57;t7NEvCy5aDux)>J@{T%pj&Ywm2}vG%yO9Zknu|c*N$FhToV`D1+FhCsrodh-m24Jt+N=a!}s$@^ukVu7S ztEd!#>=9eta?@92o?~6+A*~kCf(Vu#wu>-@B+>~XW?2&Hgs1}d=iD(IB@xIT`KrgR z_hp`!I?V$~HB#N5?2Zt_k!2*}IdHrH#Flv=I?z?&&Cm;Qe6zii2xeb&!!GqHg|f#v zL)kC@ne=mLXW+h-p)8`XPs`g_GzZFSWRmpEfJehR0@=2r$mT_|W1EIFcmMXv)=T0Jweeuxw^RAU~j&~Y|&43`Ng?1ugL3O@^ zD99^rhGiT?QAda5i`-m|aC09EWsm;W8(#KKx$Fr}m&Ndj8Y+aMKv#9lmO^tN^QCZP zYlQ_43p!9SK_oxNFpp67n7g*$A&(1AbV|ZWN)?9a4~`H)akYJjSC!1CHr_*jpb?_x z8>IpgT2m6C>`NXVm^@lKImsmnF$X4wIGLe%vV_W@EhSv#2NSfJTL;bowIG5vx+xq# zbODPxNrke<9&qW&4@#1kS4%<{9jDiC;h=?B*`f*Nhprt^|-AXZ$hO?odjgYB#Nt1J#v55_y) zlmM~`ORl{W(#D`90A5&rLWql-add&u>g^bk2xL#V_-FflP$2s%8)SRP8nvXmuC&EkmHslt$h5Ju5fF@m;KAba9=pZVSI1hTJof-Es8Z7QlCWTRB% ztK&x`0R!`F)Y`@@5L%FpiZW$WEmDE(Nr&J4`5aJ|xy(bUou&W>DS;clQq4d~WBh^I z6Z^|D4+|n_DnaLW(Woj!D5*g9<-2|O%1tuQYh31`QlxHY0z9G}Vx>y+P>)q_+73`6 zUgyp1Vdd#6#1#b*#-4oNA?ZW%*xqWDP%9u3fHx9B38q}AK zd2C7|jD5w2p7+Y%NRob+BvAdyxyY~a6M)}w0usoQ7tz_WC1C*?XADMijM616S|=06 zzViGlk33P53{*>^>RH0?q?pHnK^4;xD~Xn?CQiv=S)gO4w2UwX?HZB@V_&s!@mrrI zNd~JWp-4tf%wfXGmQv4jVJzXQN`s}EUzF-*hw5xJ5ot9U!F?u-ef8#(e~{nG7;M>NAe4rx zaSc&8_t|GQ!5mRSkyr3mb{40g0c@Edw)c!3zW*;NWu#gu!V}sM%Kq4bREAtiA*urb zZHGk?T}2YYs;lCgn^*|jx83lHtHN$??uz|ycq-S_F1t#%6TkNT&wj*PG_QNkz{upv z&0Z(Fgh0 zyV;vVOr7#EJM#Z(^&0V zb7J;Tb1kyg6dX3!VrvbePwRb&wI;qx^L?qcrnsoNmRW0bSfs@T(oMrs4(bmV?LM<+ z>xP?bxmKs&X(?w8L1EH*tF4@ns3b$J({Hu4lY#%Ow_57yf@mDbT+?s06_mR~Q(#L& zU13~{YVnG{9`e@u{MXbmU@xhSTx>f>4-Nr zTE7#psjz#D zwQ1$dmcfB}omFSYd3)!;Vb?5d@q zE?@WIS|8j-M=hfSmX2~Y%ov5XnIUdbMRbphHR-nXJK1seBF;6V1X>$y7MovbX^5|E z{SG!`n>RK;)lv}hG-kij($BO#oA0#ML-aBGPD?$!6NH+1C)JpeK(mWw1amRXy{=BHZv$=!12?)l>rWBqztZEAR8uy=s#&K-WZ=$YL+=L@R`SC36>to5$z z9a+`ew`%a{LpXw!wPxJUNEglT-g^2kHhW#1E?Kf@kM13dZ=9H%)LXXU5{;5FHs-?O za_gLC+lJwyr+4qfPy^2J=?^`4)vB@n-l@R>&UoIQE!uN>y-{B83IO&;LVTbdADwx<`9{62o7715)OH`-T3^_`?eypIPhKR9#)lS6zu2hd zFV0a826FFceo$fUsomXLtJkImb*a1y#t6u}aQfB8%&T+2kXY_ue$4b+nv3(NKWBVw zz3%K!O<~oosY_G#;+^?oqh>DpRP~cZ&*W;bg!xw_s+&?{0p z1Q5o5G=~Fp5~U)`z`rK;$jL#gtF=NPtacX!`G6FgmQyuFH{+%k&9_p!7)kP53Ge|# zR-))(NTU3F@bL}1o++)2I<=z4D}^8`OZ2NyuS+YcZ9w6(4Wz4HZs(^kl*P=B&H-v(nmga#_PA1BonqI zMf*S`#X9<=G^OENEeQ9#Amt#ntrMUJji_2qlZc+aDT%W56?=|sdQh41q@5WTbwZIF z))Z!j)JP}2y5yH}p9?AjrC*a+9`kyjEAu!lg$>I%O3_z#Jbk2JDw%SrMC@D`arBue zggXWWAP3w5O1~wQ=$aRFkA^N*fqto}66NTt&Un`sf2@w=T31IBavr51v`YCRg)xEZ zjoBwVG;gtHT;_t?Xq$#tRa~RDD!v8vJ_Aq$v#VDfamnALk#)944AV3~0_y+NnBj&~ zlBlZ2A_wk>CzU0kG>UN``a!5dsYOW|0@=}D-28%Na+3A7lN8;Rdz#dyL}t_Y69>j7 zs%>eqiCF@Td0lr@I5n8{Axe%Rh%k2Sp6wUjC6#QjRZPW7ov*{Owz$l}#?K@adGcID0q*0j+8@q1H)<7SJh6f@Rr%oWz`5+@~?L zB7~hd=a&D;N5Y(HJ4(@I`3w-6LlpGnD0FxVU3=BS?Q1IVRmB(K>;Y8pds7er?Bp#E z{puSs&uMP+a8sIUF8bwIrdnOC6lzeA31O?)N?`$Yih4yrLiNv-MG!kR^4##V8l`l) zYm^clR=6~BH`)F9p{v4g0PE@NsJUW_A%I;we#u{#NhN1ERRWrT zA~Xi_OLcAuMj+Q z91~apfhEO2+~OR@LW%C!8^ZWP+6|p^{CA&&FlT$Ny{tB^Shn;Nb+g?i3Y(02wC%QV zR#BK@S%nQ8M5ZXh+KsoZdhF}c3uC%m6JjtDr?Oa(oQV3Qie3UyCeb`+KlgnqSJ-rMjl3jSXfDQAW0)cLo;+O%-AiVzD#sZtO9L= z`=_QTLfg~B>)-VjiE^G(6d;m_iDnN$M&9nL^b{Bdz_Mr&LaC6BhlWk3Yy4kbj6xhkL|L&jyeeJ{NQf9gB~WiC$MS zQPFS7vB^0glo7+j)T;Jd%2ssUF@kL&7NDd-P{?PcSectWbm!37a z31BO?k}iNjEy5{E(r~KccVeY;h?t-o_(*e(ciXb(zg|vrami_l5CfeV&I1zB=FhL@ zRKl+AdT}7xqEHspWySnbjjm6ciU|)`Y%5O#t=~uoox`fS}jlu zV2p4v*!Eu*_yl}dQ`{hg7#c^$!YM@ZPx4Jn53|1-Vj$1zN0*#e zL&B)=aD{|heyGP3B|?<*y+bGVJx%WN2DiItm>nNPaRu6r>NJ(^LK_l{*HAOdU048i z;bK69P}*ur6C=$9-un;x*|~}s-sp-L=(CfAQo!PHV}7B73y(w4c_KGun1p!_z-eV} z2E{Q^Ww={v3X>qr>%0fP`mFqTywqtWkx|Lxsl&seC0DG-o+S#mM|C&K)C-bu>xo>E z51OW$84e@tz0g}X_KTbyd0FX9DsCgLLez;D$QW^861B?cr9nYcwOd;$ybOmT&jHBj zEqD?1UgZ7i#6Mmve|eMJUr;)rqJDI?8frQicJ3c!RHW(jlx zad-Ug6K|6=Z+1z;bvvfgL19HVI%#puD(_{QECi7JcD-dK78-a9!#F4`sDM?p67vK) zfw+g_j~*xQeM{MUix8t}0$d2;HUh)8NRl1G5L+->(~iAi{1RRg+`;MP?6>M1p^*0y zZkCG`j%EGEeMK`1=O$w0&MUE@>`z zOQT6>aOzaaWo$zA96(A8=#kz)iR26CmYoQ{m)w!unwgV^ZUn(^^y+v0@}H9C3b!;+ zd^t`~G7)w@^rX^nm>>q}RFxjfiCF++C@6_Zs(EOJG=ktuy%R1y>>ZNkN|!WjDZo<{ zYtAR~$7;$xG+rfbw3O^<2HlVp5LW3r6C2z-v_oO=W!}IZR~#;B-tLk{w^(H+`lqxr zATL&jok&BmRP=9KH!S1>+DKtUrM4_37=DxYo>Sldd`WYaTN*+LT^0lQV7Rfbs&_ba z4CdHy?Z}pf1?Cn)Eay5AL4c(jLGhctYyW)4u7aC)xZy?vhZ)w!d>Z&>g#MLo1J6gH zAf!UeZCIe_-ZgRhBxMYmz-KqrXzg=>4iEY-Zzts0HZ(#=aT-&HWHmWH+_K&iyY zl7~027$fLUoCidKq(}z-#`*av2sBAB(F@_1dc8F^8ZN zv>SQN!Fx-ZtKHHl0D$aGT}G)-^>j;W6p##b9meibq%7cVnzu{rDi*zAD53D%y;D~m zo$Ka(ZfWvxe0oA7H*(pzm4^u!F+2!!9^Z9Hj*)3u#G zP})wjvZ#Pe6amCc)9EX3fDr5{kYLtmSrp<3e4j&`;ym?qLlhzKJH1Pu^ULGpGaqzW ziF5^8G9_r;!E#Q0AsHdPB_%I4`!WJGZe^?0N1siT$coa$g5Y;~*I)C6xA-uFukrT( zu&BW|=G5Q|{_wvgYVhu}?B8i>@OIOP3PrIkH<&mTY@#qz;KSq` zUW-=+n~BV;xoGjK%teb=)y4KkxS5Z&cxPP%?(jZmc*8dF4EkZ~jTY~$3$h9XshKz0 zyff-$tq-+nXE`r#=6yEnjHzTT&Yn#Uo|8-3B-`5Rfz4BRtEC^vTxg*AR$D)Cqx_t} zB+?lXsYzMW2)ou>Ed_OjFr8b3KPKVZZ=EF>Hf$A%&fKKUVj<(5P~qa%nx&&owrF#$ zEi-tAUS~6db9!BK&C-vtW~qm@7Im{7FSund-%^gKn=KlpZ|f!}A2!9Ybd!H&bFHiz z?Sg5xv85V=4{8&G%|*+7wAiw8J7%yyOVYLUBbs3vH$B9r25W!$erwfIQO@(8%p$sAu6mmqT=bH5%-|OBl^tV< z${Snue6q~oS?w^4!1z=YFiV*YZS9chw)T5i*3RsvZNm(n{aEfR+uu1WP&;OD%Utc4 z!Ly5&f~H-wNmaHS$XvAjC*Nbucec?`%P1hA@l%Q$&}Ie~4Q3OA4Yjvp2Dg4?J7)0g zqP4}QKlb0G2E#~Ut|K=`c9^9GbI@`jrh*wPh6IrZdv;-aW-$A`(PaL%VFq`>?ard+ zA2f-(1?q4Y_Rr!cLJgk&!<}qu@ErDgx@hL*<`;;PO#HyFsvrhWFI&7|;v?MM;suj~ zEg3uY5>zRtYb zoWg2h2vF$86II(0J`0qcXq(QEgH36ayT8Yq_m{69DQQ0JmWKAWnz)QzL>Y?_Ku@_x1Vg$suskYcWPD@Ij0vB=4~`9znQIL%ZBEX@Q! z8am|Yl{z^0NG%F#tmOTD-lq=z9UgdYrKn|U%O5o{4uu&^J5?b>23j>d384`eMS?&-~pCskML%>K0q%4t_Y0@ zZlM>YVcbU3HgT!iXpXI@(U7>0lJ&=_pg^hM#T< zBMiRQd#K}c-NKnqIB_PQ_ygyJ%lRr)I~iu6PHqIwYozJ3LxDLTX`NZLXP+vV8jcRe;Y}E|C-HUU}cYw(&exD8#G5t)X2z;;e z_PyxMIkDnXE`I@IRMKN-LqFD3TnEN*Gl)}E56m)4axi(DtgXDTY_2t~+IAcR2-K~l4ZN~(cf%FGa^CPH$BX=ZFC;qOPh1Fyb0?|k0u zk_P;R1*TyNl=a~eR6lAHVlU5$AlfJ&O~i|w1tN-YirUoOL;!rFHx}IdNx{r5ZfV%i z&>*2SYYGg7+UlLnOkKc4L_AA3EO79{LZF&S<Q05}Q& z=rib!6~}iqYl-p{!-DBbT5e204vkcis~#{x@5mHJSo|q(Fj%#nEcFGqr8K~o@E)tv zU4vy)Z7C6mHSDl#DOTXOu{Ti;WO7S40^^&!Tfh3gPsmbV+>)iZ!a=`V!yTalRX%Wp zskD%*HRGjZDbbDzhGFWXVPSybH$vl2d+!>X=#+B4&-Ck|iEJLo7Jd(}O2ih*ME z>$a5bIMa9Oz%2GU0A|@raD21(p$8^6NI742D@P+Q;d9W&g54-WVx=34-ax!smWBmQ z*nuL!lqx7@rg8+ww|Iwq{WsT1IbU%rhxI99yd+Ozz9`gt3TUe&p%aY0vM|X-zkW1JD)qyxw zw|Us1L@34Wl?j$V=e_BuZ(S{uech!ax*K^WrT9Yp?&{fMaO>d+al?WG+x8&hWBRfL z>4cOVqozY=!sX9V0z#r@$53xraa2N4NtEE=4XvN$kg@(EN>L4Yz- zJ3{6!cz0fT&)u@sH{G^Ur`+qLZKMGy7f|$h6UaH=0X!XW}}+Y2$;X@J$%bU*GM_H zyOqPSNbm*Gi{c4@=)fLLAR-#((6>E@6{Ws(z)>|fJWX+g%wO@YIAPxL66X$=I4FnH zPfwKrFGQlK-sO7Y1Ghy;32kx23MM1~#4tG4G9%?o$b6f3-v_Smkt2V{Wh;RWTY@W4 zqH_ZF)rAwpkzJ{_S;`SHSZqZS3AQE~pQ#)H^H;s&ZoT_4Dd)Rx93J)zUR^n0k~In$Ro-*=0{brOi@gzTEc0Q18o4u*gqUiKV{$>LWj zWQP!EmMzdd;(XIv_=<&>N}L~TK^&O39KX!}7K&7IX-7$Ul{>5X->cue6n>=!%_N_G6wDLH(9skQz99 z-2h~wOr-g?ck6ja9U$f0zo{G{ z^6lQ|e*NwtDd+C8a%S^kLJOh+Hn=W@sDf90cp?t-bd_e>hbR0H?i@p`r}x3-LF4cw zYeJekywl&`dybUzBe!yZ3JOe=NOd!ZNVwWoTzS(0lHW}>iUqK8*-XR&M7Z<|%)=9| zDB^s_`_ZS@{Y1+7aalP<&(R~fp9q7TEqs>-cybl z$&ZJhl%+9D1&Iw6b7ZlFRl=`2%9*l9vWr9|$I(SO_@K;?av=b639bt4Wi))hUTalwyt8NH~+M5;Ca}vk>tc{%bK_NhJ~G2j18pPkUT0{EIC( z4V5kp!XPu}b{?0LDix#aaKs*^%mF4AO%zTl+?URf$!6!e4buG3o44}Dx65hnaXAgq z0PKE6CbTW{2jNujZ(WJwiWfo{xtZZ(wb+d-d*@ZYi_Bwai@6$5WZ`|TAY?ihR z2t^x8PtkFSliwID*DOTNl@yfU^b}F^njVCG#QBN$y&K+nro{QJTO2~Rn0u6*GCKH3 z3M#cDW+%Ol(j(;fEo=tiId?jQe`ZMG3yOd0{dkuz9Qs^Xxc7Sdzh5leJ8~B8S?~Mc zjrM@vG7FbbPL8e12Czn2<9nwk^mdbkfs>R{0*i*bYugdMom4KHbXkkzWfE;I7Uj(1 z47JtfFmd@na}nFk=M_niL3kA;TmlYJVe&Ux46l4&kqpCNjdj6@L;o`jL4K35idI`ZNa#Jrn#L1`E&5j*3N9lz zo9tbS0oO%X9kvLnB?_)m#MWDF1@YFXnu8k_)L^LyDl#PQCJWVOaU0ZDOGi0-Zbn0G z$MZJVEDag&vlxO!MqV2dZebp<aFIYhNm?RU83MpHg9=B zI|}aXqU}FL<+i2Z&iGcF@w|Z8jB?v>aA&Esmj4*$Y|FuIeyF9MqT>Lv3fj!(ZRxemY~I2+U`s#KO0Op1&bpROz@28iHs!Ir zr>ORJ4BXc4vl+NWTiOI%W6g4&e65mxySHO9?_AYJprr-|`|tPe1~}$js{m z@3u57a66ugETs{i^J8Z3zOwh9dtbWp5Bn>+|Ghi9XI20WDH3n;8k$!QJqSSqq$FChAcVz=f()A6+S-#awzG z7F;Q{wt zoxWHU^OeK@#!DXC^$E|;c7Q$js@_>q2j`+0bX!*y9kecOAS*D116 zTcg~fev&DSGWg$mUw?MT0oqi5cBuwo)K*}ohsbt%Kxd`jz$swZ23&xWG_2?$II7Q4 z88aCom=zxbpRQe(r&3Iz->a|BIP4y_JD$> zr2@h6@4T_wZhV!L^H5niMN82MM=L6Fu+4eMTV>6V8>TdJ)pEB@#R4O4AZe=WLQTpr zRX)$_KJTx`UH&IY^B0#is@lN95%W#+`e=~4q{%DoWygVrz<#6?1uKgmXYQ#$X#9Kc z(%ZiLB}wzYZfWw#D{4s57hnrJ;Fz*09XZI8c9{-(ENv*JU(m$Cku7P2#`k;Cn;M^& zG=D8iQ{3U~Zp~fO^eT2|HPMq`niEX~_L8M2r}9uvjw+3|XM35J5*q)(yZBEV->n_~ zaCwK%N<*asuxF-+cpVhKtJ~KKw3wAq?z1T?RhLbSXV?ZNK^UDQ%md!V&v;2r5dND> zH99t_T8ntY)D#U)sNCU{Q)y3&b8b7aI0%&ZL4ySQ`st(gZG{q4T|Bv_hOWv0U#{Y0DhxV!Z6R1bC zkzC>s|MJW^F|90W6Wmz3Ed(T#tf7XPNSx32ddQnNbmfP&zaQVi{-)ojVfV1_2>q&~ z8xm<+Gss(N6(bCH8Ytz%Rpt*&=nDnJfAPY*79S`}{nKSBf*afyY9}BeLMee%)duIP zE2#mLk>2>!#?&L{A*L%n2iUBO-9PaUK`T zmbRf37DM&|_t@*iV#Pfb3W)#e{e7#Whqb59>TphRWfTlod+2_Xy+BW>?iBNI2#^>G zLCJ}UiXkdMw9+GI!7fu8A@RfBhraTS^-|5*9nP7|n(CA5IN#_l)u^sd^Q&}XQoiI4 z0m%eG0CR3Fw-FHk&3ouCeeaRmZ0>OK{~<;3*axBkn9s`3s@+DH&>(%3GhfQ3v~Sdr zVwy1=Da&mH#*cV^`sd}xN}Sia#K}j?fLV|yP&BN}YE~1^teC#FP(B-2(I{886s}`k z-J^kuIDhwcTY6U>h@RsThwKhG2S?cFaB(S9Rcl8qTri0^(}q9dV$NH}PjV@j5r3w3 z@(Ev$df)%*{9UA-bGIZ8`{0RCcbF%WbKZy!%i`!#7!6(og85)Naz5hgG4Ib`JNaCR zbDm2ak_OtGFwh7A`4>`{YV9ceCFO2&a%zBu`hg@2YT{brs7~<@@BBLtxmV(x?-EDl zeR_ukH=57K|5S&fJV7EatCX2U)bGjT0Z|yM(8350i=Yz;kRSK1e{T1k0-Xz7pd)eE z=6c2w9b<8fs}FMGU*d*xoq~pep2ncr(;EC_MLCi1_@Ca=kN%)8XBrWlJ7DsUBn zNwu6d99q?;O1S|*kXWWDBTddsfpSkmuIDirx)C7%%X|10yL?=xy3lPZOr(Khgx7!$ zuv0A#WLu4zY7+vo!c8?Uv2c|N)`(>)A@W%r7rv~KXTKMfl~X*d$U1afX|ScLfDY{t z0s{lgzAl|stk7R(hbz>9z+xyzkbHK>p4XlDkj{tKm(Pcysg$-dp^$7`EVAld#Zx6n z+sXY)VFh($uunuJ$_pHOrZ58J%^fRFz2uQS5$56!$QDcHh2uoF>!YmS@7u7icd55E z#+-kVx8eBH-m?!K_e(nVf0sJ$zsfuAfByXk-qEqeN8)mUE5Bn*X}|nPT+R`KFd|i} z-#*jfwi~9pS=(%4Tm8}}(r}+?e_Krp&E;WCL{-OKSBsmAR?AE8OjZA`YPs{VGrz=c znu_AoT(f$nv~(Z<%yi@Jjt`P3*Z>ys^bNuv_k|wPrz4HZi@x_)uFj>Zi>}V|UyQ zY3!Ce-`G;AJ@tyUR*Sa$G+bI#t#7n+!#4)4CxDi^?L>hs@36b=U39ebEj!)Dw|s|3 z@aCIqdePEPQLk;Dh1;Sz*v)pB9J90E+I8DomMnGK$?lpez;COq_5ZcoKI8v(x1FEg z@_^E~D_pK+k2cMAMw>NT)Hd|wPS;$tkQp=2er&sD`|QWMwDTmJ?S-!_D8)PR~kx>1Z`$rb!iY*xM+#%A-FStQ`u`*{k^i+&I5+%^l#}IxcOiG{-zsu(xcS= zyJOL*=EQ;jXn6a~`)=L))(#%K(Q(L4UEOo1HjIWNemcBh+tnkZBg@C9*7=LJoB7vp z={kQ&=ZdkuXCKHvmJUx%jZZFHwru&x)bQHATL0MUWh>T>_74w^Ho|2`^X|isJ$`Ap zY-nV3aBA)7;IgTS-jUJT_(p&6ASo=Rf^2}ttT1fC?;A?{{DHpSH0}+O-Y7o8JJvg8 z>s34$dUC4o@{YOQf}Pj*jt=xKotzq*=v_XzbbM^pNdLyA>l!%t_A@WOxOd~&+NlL| zmYnD>S+cHs=ZV2JYeyyqR}YR(P1c4c#x@O()}}U0E!buGQT<08vEr12*RMV@J%03s zJp(KM>xkzZ{JiHa{>~Er`8~s9s|S1ZBUaRx_tf-w>Qz0H!@U!OJ-G-yy%SR-L%sb| zlRcADy~{^Nm-kGq9v=)gCF%0XV9Dz4r%X$Vf@&)!$3_?IlC68u(b>s|9N8F*)81&+ zF+)cja>7b|-8pmr-LH!_EE`x}UlEOao6hn!O>G#9$3{n2hLaORQ=>^Tu_>O4M(Z0l zto4_VBvJ3^)ac4}tHWsBim9>Dbor*`<3r*44QuVZw`#x|^9F|vB&rd30JG~QS-7)NUwt5$3pUPGfNTeEImV_;}vYTcTN<@IE< zca}H0ad{&fB>@{5*|c$TvUl~SkriXB{E@L$gPVpo4JDKP!^7*=`5VSZ8u8$8wl)ga zNAbqEx3_<+cjefM;nmCIflc*rQ?_<^{h&WJvI!Z+*9@$kXpF5{Qy&-}_OnfE`jhe1 bel*#)ZcXp1cyeec8`=;xR<0dLv*rH}dbB%s diff --git a/wandb/run-20231023_091030-25yuqsbo/files/conda-environment.yaml b/wandb/run-20231023_091030-25yuqsbo/files/conda-environment.yaml deleted file mode 100644 index cfced9e..0000000 --- a/wandb/run-20231023_091030-25yuqsbo/files/conda-environment.yaml +++ /dev/null @@ -1,297 +0,0 @@ -name: eye -channels: - - pytorch - - defaults -dependencies: - - _libgcc_mutex=0.1=main - - _openmp_mutex=5.1=1_gnu - - blas=1.0=mkl - - brotlipy=0.7.0=py39h27cfd23_1003 - - bzip2=1.0.8=h7b6447c_0 - - ca-certificates=2023.01.10=h06a4308_0 - - certifi=2022.12.7=py39h06a4308_0 - - cffi=1.15.1=py39h5eee18b_3 - - charset-normalizer=2.0.4=pyhd3eb1b0_0 - - cryptography=39.0.1=py39h9ce1e76_0 - - cudatoolkit=11.3.1=h2bc3f7f_2 - - ffmpeg=4.3=hf484d3e_0 - - flit-core=3.8.0=py39h06a4308_0 - - freetype=2.12.1=h4a9f257_0 - - giflib=5.2.1=h5eee18b_3 - - gmp=6.2.1=h295c915_3 - - gnutls=3.6.15=he1e5248_0 - - idna=3.4=py39h06a4308_0 - - intel-openmp=2021.4.0=h06a4308_3561 - - jpeg=9e=h5eee18b_1 - - lame=3.100=h7b6447c_0 - - lcms2=2.12=h3be6417_0 - - ld_impl_linux-64=2.38=h1181459_1 - - lerc=3.0=h295c915_0 - - libdeflate=1.17=h5eee18b_0 - - libffi=3.4.2=h6a678d5_6 - - libgcc-ng=11.2.0=h1234567_1 - - libgomp=11.2.0=h1234567_1 - - libiconv=1.16=h7f8727e_2 - - libidn2=2.3.2=h7f8727e_0 - - libpng=1.6.39=h5eee18b_0 - - libstdcxx-ng=11.2.0=h1234567_1 - - libtasn1=4.19.0=h5eee18b_0 - - libtiff=4.5.0=h6a678d5_2 - - libunistring=0.9.10=h27cfd23_0 - - libwebp=1.2.4=h11a3e52_1 - - libwebp-base=1.2.4=h5eee18b_1 - - lz4-c=1.9.4=h6a678d5_0 - - mkl=2021.4.0=h06a4308_640 - - mkl-service=2.4.0=py39h7f8727e_0 - - mkl_fft=1.3.1=py39hd3c417c_0 - - mkl_random=1.2.2=py39h51133e4_0 - - ncurses=6.4=h6a678d5_0 - - nettle=3.7.3=hbbd107a_1 - - numpy=1.23.5=py39h14f4228_0 - - numpy-base=1.23.5=py39h31eccc5_0 - - openh264=2.1.1=h4ff587b_0 - - openssl=1.1.1t=h7f8727e_0 - - pillow=9.4.0=py39h6a678d5_0 - - pip=23.0.1=py39h06a4308_0 - - pycparser=2.21=pyhd3eb1b0_0 - - pyopenssl=23.0.0=py39h06a4308_0 - - pysocks=1.7.1=py39h06a4308_0 - - python=3.9.16=h7a1cb2a_2 - - pytorch-mutex=1.0=cuda - - readline=8.2=h5eee18b_0 - - requests=2.28.1=py39h06a4308_1 - - setuptools=65.6.3=py39h06a4308_0 - - six=1.16.0=pyhd3eb1b0_1 - - sqlite=3.41.1=h5eee18b_0 - - tk=8.6.12=h1ccaba5_0 - - torchaudio=0.12.1=py39_cu113 - - torchvision=0.13.1=py39_cu113 - - typing_extensions=4.4.0=py39h06a4308_0 - - urllib3=1.26.15=py39h06a4308_0 - - wheel=0.38.4=py39h06a4308_0 - - xz=5.2.10=h5eee18b_1 - - zlib=1.2.13=h5eee18b_0 - - zstd=1.5.4=hc292b87_0 - - pip: - - accelerate==0.20.3 - - aiofiles==23.1.0 - - aiohttp==3.8.4 - - aiosignal==1.3.1 - - altair==4.2.2 - - antlr4-python3-runtime==4.9.3 - - anyio==3.6.2 - - appdirs==1.4.4 - - argon2-cffi==21.3.0 - - argon2-cffi-bindings==21.2.0 - - arrow==1.2.3 - - asttokens==2.2.1 - - async-timeout==4.0.2 - - attrs==22.2.0 - - backcall==0.2.0 - - beautifulsoup4==4.12.2 - - bitsandbytes==0.37.0 - - bleach==6.0.0 - - blis==0.7.9 - - braceexpand==0.1.7 - - catalogue==2.0.8 - - cchardet==2.1.7 - - chardet==5.1.0 - - click==8.1.3 - - cloudpickle==2.2.1 - - cmake==3.26.1 - - comm==0.1.3 - - confection==0.0.4 - - contourpy==1.0.7 - - cycler==0.11.0 - - cymem==2.0.7 - - datasets==2.13.1 - - debugpy==1.6.7 - - decorator==5.1.1 - - decord==0.6.0 - - defusedxml==0.7.1 - - dill==0.3.6 - - docker-pycreds==0.4.0 - - einops==0.6.0 - - entrypoints==0.4 - - executing==1.2.0 - - fairscale==0.4.13 - - fastapi==0.95.0 - - fastjsonschema==2.16.3 - - ffmpy==0.3.0 - - filelock==3.9.0 - - fire==0.5.0 - - fonttools==4.38.0 - - fqdn==1.5.1 - - frozenlist==1.3.3 - - fsspec==2023.4.0 - - gitdb==4.0.10 - - gitpython==3.1.31 - - gradio==3.24.1 - - gradio-client==0.0.8 - - h11==0.14.0 - - httpcore==0.16.3 - - httpx==0.23.3 - - huggingface-hub==0.16.4 - - imageio==2.31.1 - - importlib-metadata==6.2.0 - - importlib-resources==5.12.0 - - iopath==0.1.10 - - ipykernel==6.22.0 - - ipython==8.12.0 - - ipython-genutils==0.2.0 - - isoduration==20.11.0 - - jedi==0.18.2 - - jinja2==3.1.2 - - joblib==1.2.0 - - jsonpointer==2.3 - - jsonschema==4.17.3 - - jupyter-client==8.1.0 - - jupyter-core==5.3.0 - - jupyter-events==0.6.3 - - jupyter-server==2.5.0 - - jupyter-server-terminals==0.4.4 - - jupyterlab-pygments==0.2.2 - - kiwisolver==1.4.4 - - langcodes==3.3.0 - - lazy-loader==0.2 - - linkify-it-py==2.0.0 - - lit==16.0.0 - - llvmlite==0.39.1 - - loralib==0.1.1 - - markdown-it-py==2.2.0 - - markupsafe==2.1.2 - - matplotlib==3.7.0 - - matplotlib-inline==0.1.6 - - mdit-py-plugins==0.3.3 - - mdurl==0.1.2 - - mistune==2.0.5 - - mpmath==1.3.0 - - multidict==6.0.4 - - multiprocess==0.70.14 - - murmurhash==1.0.9 - - nbclassic==0.5.5 - - nbclient==0.7.3 - - nbconvert==7.3.0 - - nbformat==5.8.0 - - nest-asyncio==1.5.6 - - networkx==3.1 - - nltk==3.8.1 - - notebook==6.5.4 - - notebook-shim==0.2.2 - - numba==0.56.4 - - nvidia-cublas-cu11==11.10.3.66 - - nvidia-cuda-cupti-cu11==11.7.101 - - nvidia-cuda-nvrtc-cu11==11.7.99 - - nvidia-cuda-runtime-cu11==11.7.99 - - nvidia-cudnn-cu11==8.5.0.96 - - nvidia-cufft-cu11==10.9.0.58 - - nvidia-curand-cu11==10.2.10.91 - - nvidia-cusolver-cu11==11.4.0.1 - - nvidia-cusparse-cu11==11.7.4.91 - - nvidia-nccl-cu11==2.14.3 - - nvidia-nvtx-cu11==11.7.91 - - omegaconf==2.3.0 - - openai==0.27.0 - - opencv-python==4.7.0.72 - - orjson==3.8.9 - - packaging==23.0 - - pandas==2.0.0 - - pandocfilters==1.5.0 - - parso==0.8.3 - - pathlib==1.0.1 - - pathtools==0.1.2 - - pathy==0.10.1 - - peft==0.2.0 - - pexpect==4.8.0 - - pickleshare==0.7.5 - - platformdirs==3.2.0 - - policies==0.4.2 - - portalocker==2.7.0 - - preshed==3.0.8 - - progressbar2==4.2.0 - - prometheus-client==0.16.0 - - prompt-toolkit==3.0.38 - - protobuf==4.22.1 - - psutil==5.9.4 - - ptyprocess==0.7.0 - - pure-eval==0.2.2 - - pyarrow==12.0.1 - - pycocoevalcap==1.2 - - pycocotools==2.0.6 - - pycryptodome==3.17 - - pydantic==1.10.7 - - pydub==0.25.1 - - pygments==2.14.0 - - pynndescent==0.5.8 - - pyparsing==3.0.9 - - pyrsistent==0.19.3 - - python-dateutil==2.8.2 - - python-json-logger==2.0.7 - - python-multipart==0.0.6 - - python-utils==3.7.0 - - pytz==2023.3 - - pywavelets==1.4.1 - - pyyaml==6.0 - - pyzmq==25.0.2 - - regex==2022.10.31 - - rfc3339-validator==0.1.4 - - rfc3986==1.5.0 - - rfc3986-validator==0.1.1 - - safetensors==0.3.1 - - scikit-image==0.21.0 - - scikit-learn==1.2.2 - - scipy==1.10.1 - - seaborn==0.12.2 - - semantic-version==2.10.0 - - send2trash==1.8.0 - - sentence-transformers==2.2.2 - - sentencepiece==0.1.97 - - sentry-sdk==1.19.1 - - setproctitle==1.3.2 - - smart-open==6.3.0 - - smmap==5.0.0 - - sniffio==1.3.0 - - soupsieve==2.4 - - spacy==3.5.1 - - spacy-legacy==3.0.12 - - spacy-loggers==1.0.4 - - srsly==2.4.6 - - stack-data==0.6.2 - - starlette==0.26.1 - - submitit==1.4.5 - - sympy==1.11.1 - - tenacity==8.2.2 - - termcolor==2.3.0 - - terminado==0.17.1 - - thinc==8.1.9 - - threadpoolctl==3.1.0 - - tifffile==2023.4.12 - - timm==0.6.13 - - tinycss2==1.2.1 - - tokenizers==0.13.3 - - toolz==0.12.0 - - torch==2.0.0 - - tornado==6.2 - - tqdm==4.64.1 - - traitlets==5.9.0 - - transformers==4.31.0 - - triton==2.0.0 - - typer==0.7.0 - - tzdata==2023.3 - - uc-micro-py==1.0.1 - - umap-learn==0.5.3 - - uri-template==1.2.0 - - uvicorn==0.21.1 - - visual-genome==1.1.1 - - wandb==0.14.2 - - wasabi==1.1.1 - - wcwidth==0.2.6 - - webcolors==1.13 - - webdataset==0.2.48 - - webencodings==0.5.1 - - websocket-client==1.5.1 - - websockets==11.0.1 - - xxhash==3.2.0 - - yarl==1.8.2 - - zipp==3.14.0 -prefix: /home/chenj0g/anaconda3/envs/eye diff --git a/wandb/run-20231023_091030-25yuqsbo/files/config.yaml b/wandb/run-20231023_091030-25yuqsbo/files/config.yaml deleted file mode 100644 index a263d39..0000000 --- a/wandb/run-20231023_091030-25yuqsbo/files/config.yaml +++ /dev/null @@ -1,38 +0,0 @@ -wandb_version: 1 - -_wandb: - desc: null - value: - python_version: 3.9.16 - cli_version: 0.14.2 - framework: huggingface - huggingface_version: 4.31.0 - is_jupyter_run: false - is_kaggle_kernel: false - start_time: 1698041430.036499 - t: - 1: - - 1 - - 11 - - 41 - - 49 - - 55 - - 63 - - 71 - 2: - - 1 - - 11 - - 41 - - 49 - - 55 - - 63 - - 71 - 3: - - 1 - - 13 - - 23 - 4: 3.9.16 - 5: 0.14.2 - 6: 4.31.0 - 8: - - 5 diff --git a/wandb/run-20231023_091030-25yuqsbo/files/output.log b/wandb/run-20231023_091030-25yuqsbo/files/output.log deleted file mode 100644 index 2f6a9a3..0000000 --- a/wandb/run-20231023_091030-25yuqsbo/files/output.log +++ /dev/null @@ -1,161 +0,0 @@ - -2023-10-23 09:10:36,980 [INFO] Start training -batch sizes [[2]] -module.llama_model.base_model.model.model.layers.0.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.0.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.0.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.0.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.1.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.1.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.1.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.1.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.2.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.2.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.2.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.2.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.3.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.3.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.3.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.3.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.4.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.4.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.4.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.4.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.5.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.5.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.5.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.5.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.6.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.6.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.6.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.6.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.7.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.7.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.7.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.7.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.8.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.8.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.8.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.8.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.9.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.9.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.9.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.9.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.10.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.10.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.10.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.10.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.11.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.11.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.11.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.11.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.12.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.12.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.12.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.12.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.13.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.13.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.13.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.13.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.14.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.14.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.14.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.14.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.15.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.15.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.15.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.15.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.16.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.16.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.16.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.16.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.17.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.17.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.17.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.17.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.18.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.18.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.18.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.18.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.19.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.19.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.19.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.19.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.20.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.20.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.20.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.20.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.21.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.21.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.21.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.21.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.22.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.22.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.22.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.22.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.23.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.23.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.23.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.23.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.24.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.24.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.24.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.24.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.25.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.25.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.25.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.25.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.26.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.26.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.26.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.26.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.27.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.27.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.27.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.27.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.28.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.28.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.28.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.28.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.29.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.29.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.29.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.29.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.30.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.30.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.30.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.30.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.31.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.31.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.31.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.31.self_attn.v_proj.lora_B.weight -module.llama_proj.weight -module.llama_proj.bias -2023-10-23 09:10:39,273 [INFO] dataset_ratios not specified, datasets will be concatenated (map-style datasets) or chained (webdataset.DataPipeline). -2023-10-23 09:10:39,273 [INFO] Loaded 12171 records for train split from the dataset. -2023-10-23 09:10:39,291 [INFO] number of trainable parameters: 56627200 -2023-10-23 09:10:39,292 [INFO] Start training epoch 0, 1000 iters per inner epoch. -Train: data epoch: [0] [ 0/1000] eta: 0:54:33 lr: 0.000001 loss: 1.4162 time: 3.2736 data: 0.0000 max mem: 33043 -/home/chenj0g/anaconda3/envs/eye/lib/python3.9/site-packages/torch/utils/checkpoint.py:31: UserWarning: None of the inputs have requires_grad=True. Gradients will be None - warnings.warn("None of the inputs have requires_grad=True. Gradients will be None") -Train: data epoch: [0] [ 50/1000] eta: 0:08:41 lr: 0.000001 loss: 1.4426 time: 0.4963 data: 0.0000 max mem: 38128 -Train: data epoch: [0] [ 100/1000] eta: 0:07:50 lr: 0.000002 loss: 1.3406 time: 0.4881 data: 0.0000 max mem: 40846 -Train: data epoch: [0] [ 150/1000] eta: 0:07:14 lr: 0.000002 loss: 1.5900 time: 0.4890 data: 0.0000 max mem: 41228 -Train: data epoch: [0] [ 200/1000] eta: 0:06:45 lr: 0.000003 loss: 1.6308 time: 0.5138 data: 0.0000 max mem: 41228 -Train: data epoch: [0] [ 250/1000] eta: 0:06:18 lr: 0.000003 loss: 1.4190 time: 0.5009 data: 0.0000 max mem: 41228 -Traceback (most recent call last): - File "/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/train.py", line 113, in - File "/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/train.py", line 109, in main - File "/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/minigpt4/runners/runner_base.py", line 377, in train - train_stats = self.train_epoch(cur_epoch) - File "/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/minigpt4/runners/runner_base.py", line 437, in train_epoch - return self.task.train_epoch( - File "/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/minigpt4/tasks/base_task.py", line 116, in train_epoch - return self._train_inner_loop( - File "/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/minigpt4/tasks/base_task.py", line 225, in _train_inner_loop - scaler.scale(loss).backward() - File "/home/chenj0g/anaconda3/envs/eye/lib/python3.9/site-packages/torch/_tensor.py", line 487, in backward - torch.autograd.backward( - File "/home/chenj0g/anaconda3/envs/eye/lib/python3.9/site-packages/torch/autograd/__init__.py", line 200, in backward - Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass -KeyboardInterrupt \ No newline at end of file diff --git a/wandb/run-20231023_091030-25yuqsbo/files/requirements.txt b/wandb/run-20231023_091030-25yuqsbo/files/requirements.txt deleted file mode 100644 index 75bc393..0000000 --- a/wandb/run-20231023_091030-25yuqsbo/files/requirements.txt +++ /dev/null @@ -1,246 +0,0 @@ -accelerate==0.20.3 -aiofiles==23.2.1 -aiohttp==3.8.4 -aiosignal==1.3.1 -altair==5.1.2 -antlr4-python3-runtime==4.9.3 -anyio==3.6.2 -appdirs==1.4.4 -argon2-cffi-bindings==21.2.0 -argon2-cffi==21.3.0 -arrow==1.2.3 -asttokens==2.2.1 -async-timeout==4.0.2 -attrs==22.2.0 -backcall==0.2.0 -beautifulsoup4==4.12.2 -bitsandbytes==0.37.0 -bleach==6.0.0 -blis==0.7.9 -braceexpand==0.1.7 -brotlipy==0.7.0 -catalogue==2.0.8 -cchardet==2.1.7 -certifi==2022.12.7 -cffi==1.15.1 -chardet==5.1.0 -charset-normalizer==2.0.4 -click==8.1.3 -cloudpickle==2.2.1 -cmake==3.26.1 -comm==0.1.3 -confection==0.0.4 -contourpy==1.0.7 -cryptography==39.0.1 -cycler==0.11.0 -cymem==2.0.7 -datasets==2.13.1 -debugpy==1.6.7 -decorator==5.1.1 -decord==0.6.0 -defusedxml==0.7.1 -dill==0.3.6 -docker-pycreds==0.4.0 -einops==0.6.0 -entrypoints==0.4 -executing==1.2.0 -fairscale==0.4.13 -fastapi==0.95.0 -fastjsonschema==2.16.3 -ffmpy==0.3.1 -filelock==3.9.0 -fire==0.5.0 -flit-core==3.8.0 -fonttools==4.38.0 -fqdn==1.5.1 -frozenlist==1.3.3 -fsspec==2023.9.2 -gitdb==4.0.10 -gitpython==3.1.31 -gradio-client==0.6.0 -gradio==3.24.1 -h11==0.14.0 -httpcore==0.18.0 -httpx==0.25.0 -huggingface-hub==0.18.0 -idna==3.4 -imageio==2.31.1 -importlib-metadata==6.2.0 -importlib-resources==5.12.0 -iopath==0.1.10 -ipykernel==6.22.0 -ipython-genutils==0.2.0 -ipython==8.12.0 -isoduration==20.11.0 -jedi==0.18.2 -jinja2==3.1.2 -joblib==1.2.0 -jsonpointer==2.3 -jsonschema==4.17.3 -jupyter-client==8.1.0 -jupyter-core==5.3.0 -jupyter-events==0.6.3 -jupyter-server-terminals==0.4.4 -jupyter-server==2.5.0 -jupyterlab-pygments==0.2.2 -kiwisolver==1.4.4 -langcodes==3.3.0 -lazy-loader==0.2 -linkify-it-py==2.0.2 -lit==16.0.0 -llvmlite==0.39.1 -loralib==0.1.1 -markdown-it-py==2.2.0 -markupsafe==2.1.2 -matplotlib-inline==0.1.6 -matplotlib==3.7.0 -mdit-py-plugins==0.3.3 -mdurl==0.1.2 -mistune==2.0.5 -mkl-fft==1.3.1 -mkl-random==1.2.2 -mkl-service==2.4.0 -mpmath==1.3.0 -multidict==6.0.4 -multiprocess==0.70.14 -murmurhash==1.0.9 -nbclassic==0.5.5 -nbclient==0.7.3 -nbconvert==7.3.0 -nbformat==5.8.0 -nest-asyncio==1.5.6 -networkx==3.1 -nltk==3.8.1 -notebook-shim==0.2.2 -notebook==6.5.4 -numba==0.56.4 -numpy==1.23.5 -nvidia-cublas-cu11==11.10.3.66 -nvidia-cuda-cupti-cu11==11.7.101 -nvidia-cuda-nvrtc-cu11==11.7.99 -nvidia-cuda-runtime-cu11==11.7.99 -nvidia-cudnn-cu11==8.5.0.96 -nvidia-cufft-cu11==10.9.0.58 -nvidia-curand-cu11==10.2.10.91 -nvidia-cusolver-cu11==11.4.0.1 -nvidia-cusparse-cu11==11.7.4.91 -nvidia-nccl-cu11==2.14.3 -nvidia-nvtx-cu11==11.7.91 -omegaconf==2.3.0 -openai==0.27.0 -opencv-python==4.7.0.72 -orjson==3.9.8 -packaging==23.0 -pandas==2.0.0 -pandocfilters==1.5.0 -parso==0.8.3 -pathlib==1.0.1 -pathtools==0.1.2 -pathy==0.10.1 -peft==0.2.0 -pexpect==4.8.0 -pickleshare==0.7.5 -pillow==9.4.0 -pip==23.0.1 -platformdirs==3.2.0 -policies==0.4.2 -portalocker==2.7.0 -preshed==3.0.8 -progressbar2==4.2.0 -prometheus-client==0.16.0 -prompt-toolkit==3.0.38 -protobuf==4.22.1 -psutil==5.9.4 -ptyprocess==0.7.0 -pure-eval==0.2.2 -pyarrow==12.0.1 -pycocoevalcap==1.2 -pycocotools==2.0.6 -pycparser==2.21 -pycryptodome==3.17 -pydantic==1.10.7 -pydub==0.25.1 -pygments==2.14.0 -pynndescent==0.5.8 -pyopenssl==23.0.0 -pyparsing==3.0.9 -pyrsistent==0.19.3 -pysocks==1.7.1 -python-dateutil==2.8.2 -python-json-logger==2.0.7 -python-multipart==0.0.6 -python-utils==3.7.0 -pytz==2023.3 -pywavelets==1.4.1 -pyyaml==6.0 -pyzmq==25.0.2 -regex==2022.10.31 -requests==2.28.1 -rfc3339-validator==0.1.4 -rfc3986-validator==0.1.1 -rfc3986==1.5.0 -rpe-index==1.2.0 -safetensors==0.3.1 -scikit-image==0.21.0 -scikit-learn==1.2.2 -scipy==1.10.1 -seaborn==0.12.2 -semantic-version==2.10.0 -send2trash==1.8.0 -sentence-transformers==2.2.2 -sentencepiece==0.1.97 -sentry-sdk==1.19.1 -setproctitle==1.3.2 -setuptools==65.6.3 -six==1.16.0 -smart-open==6.3.0 -smmap==5.0.0 -sniffio==1.3.0 -soupsieve==2.4 -spacy-legacy==3.0.12 -spacy-loggers==1.0.4 -spacy==3.5.1 -srsly==2.4.6 -stack-data==0.6.2 -starlette==0.26.1 -submitit==1.4.5 -sympy==1.11.1 -tenacity==8.2.2 -termcolor==2.3.0 -terminado==0.17.1 -thinc==8.1.9 -threadpoolctl==3.1.0 -tifffile==2023.4.12 -timm==0.6.13 -tinycss2==1.2.1 -tokenizers==0.13.3 -toolz==0.12.0 -torch==2.0.0 -torchaudio==0.12.1 -torchvision==0.13.1 -tornado==6.2 -tqdm==4.64.1 -traitlets==5.9.0 -transformers==4.31.0 -triton==2.0.0 -typer==0.7.0 -typing-extensions==4.4.0 -tzdata==2023.3 -uc-micro-py==1.0.2 -umap-learn==0.5.3 -uri-template==1.2.0 -urllib3==1.26.15 -uvicorn==0.21.1 -visual-genome==1.1.1 -wandb==0.14.2 -wasabi==1.1.1 -wcwidth==0.2.6 -webcolors==1.13 -webdataset==0.2.48 -webencodings==0.5.1 -websocket-client==1.5.1 -websockets==11.0.1 -wheel==0.38.4 -xxhash==3.2.0 -yarl==1.8.2 -zipp==3.14.0 \ No newline at end of file diff --git a/wandb/run-20231023_091030-25yuqsbo/files/wandb-metadata.json b/wandb/run-20231023_091030-25yuqsbo/files/wandb-metadata.json deleted file mode 100644 index 1511845..0000000 --- a/wandb/run-20231023_091030-25yuqsbo/files/wandb-metadata.json +++ /dev/null @@ -1,374 +0,0 @@ -{ - "os": "Linux-3.10.0-1160.76.1.el7.x86_64-x86_64-with-glibc2.17", - "python": "3.9.16", - "heartbeatAt": "2023-10-23T06:10:31.385390", - "startedAt": "2023-10-23T06:10:30.007205", - "docker": null, - "cuda": null, - "args": [ - "--job_name", - "minigpt4_v2_test", - "--cfg-path", - "train_configs/minigpt_v2_finetune.yaml" - ], - "state": "running", - "program": "/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/train.py", - "codePath": "train.py", - "git": { - "remote": "https://github.com/junchen14/MiniGPT-4_finetune", - "commit": "5c3ec8bb730b2e9df23b4bb3eb24108f9c04a8d6" - }, - "email": "jun.chen@kaust.edu.sa", - "root": "/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune", - "host": "gpu201-16-l", - "username": "chenj0g", - "executable": "/home/chenj0g/anaconda3/envs/eye/bin/python", - "cpu_count": 64, - "cpu_count_logical": 64, - "cpu_freq": { - "current": 1996.2770000000016, - "min": 0.0, - "max": 0.0 - }, - "cpu_freq_per_core": [ - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - } - ], - "disk": { - "total": 251.85773849487305, - "used": 7.905696868896484 - }, - "gpu": "NVIDIA A100-SXM4-80GB", - "gpu_count": 2, - "gpu_devices": [ - { - "name": "NVIDIA A100-SXM4-80GB", - "memory_total": 85899345920 - }, - { - "name": "NVIDIA A100-SXM4-80GB", - "memory_total": 85899345920 - } - ], - "memory": { - "total": 503.7154769897461 - } -} diff --git a/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json b/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json deleted file mode 100644 index 4902901..0000000 --- a/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json +++ /dev/null @@ -1 +0,0 @@ -{"epoch": 0, "loss": 1.2293027639389038, "_timestamp": 1698041570.1777537, "_runtime": 140.1412546634674, "_step": 258, "_wandb": {"runtime": 139}} \ No newline at end of file diff --git a/wandb/run-20231023_091030-25yuqsbo/logs/debug-internal.log b/wandb/run-20231023_091030-25yuqsbo/logs/debug-internal.log deleted file mode 100644 index 5f307dc..0000000 --- a/wandb/run-20231023_091030-25yuqsbo/logs/debug-internal.log +++ /dev/null @@ -1,1367 +0,0 @@ -2023-10-23 09:10:30,037 INFO StreamThr :54598 [internal.py:wandb_internal():86] W&B internal server running at pid: 54598, started at: 2023-10-23 09:10:30.036505 -2023-10-23 09:10:30,038 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: status -2023-10-23 09:10:30,038 INFO WriterThread:54598 [datastore.py:open_for_write():85] open: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/run-25yuqsbo.wandb -2023-10-23 09:10:30,040 DEBUG SenderThread:54598 [sender.py:send():375] send: header -2023-10-23 09:10:30,457 DEBUG SenderThread:54598 [sender.py:send():375] send: run -2023-10-23 09:10:31,051 INFO SenderThread:54598 [dir_watcher.py:__init__():219] watching files in: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files -2023-10-23 09:10:31,051 INFO SenderThread:54598 [sender.py:_start_run_threads():1124] run started: 25yuqsbo with start time 1698041430.036499 -2023-10-23 09:10:31,052 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:31,052 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:31,053 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: check_version -2023-10-23 09:10:31,053 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: check_version -2023-10-23 09:10:31,364 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: run_start -2023-10-23 09:10:31,369 DEBUG HandlerThread:54598 [system_info.py:__init__():31] System info init -2023-10-23 09:10:31,369 DEBUG HandlerThread:54598 [system_info.py:__init__():46] System info init done -2023-10-23 09:10:31,369 INFO HandlerThread:54598 [system_monitor.py:start():181] Starting system monitor -2023-10-23 09:10:31,369 INFO SystemMonitor:54598 [system_monitor.py:_start():145] Starting system asset monitoring threads -2023-10-23 09:10:31,369 INFO HandlerThread:54598 [system_monitor.py:probe():201] Collecting system info -2023-10-23 09:10:31,370 INFO SystemMonitor:54598 [interfaces.py:start():190] Started cpu monitoring -2023-10-23 09:10:31,370 INFO SystemMonitor:54598 [interfaces.py:start():190] Started disk monitoring -2023-10-23 09:10:31,371 INFO SystemMonitor:54598 [interfaces.py:start():190] Started gpu monitoring -2023-10-23 09:10:31,371 INFO SystemMonitor:54598 [interfaces.py:start():190] Started memory monitoring -2023-10-23 09:10:31,371 INFO SystemMonitor:54598 [interfaces.py:start():190] Started network monitoring -2023-10-23 09:10:31,385 DEBUG HandlerThread:54598 [system_info.py:probe():195] Probing system -2023-10-23 09:10:31,390 DEBUG HandlerThread:54598 [system_info.py:_probe_git():180] Probing git -2023-10-23 09:10:31,403 DEBUG HandlerThread:54598 [system_info.py:_probe_git():188] Probing git done -2023-10-23 09:10:31,403 DEBUG HandlerThread:54598 [system_info.py:probe():240] Probing system done -2023-10-23 09:10:31,404 DEBUG HandlerThread:54598 [system_monitor.py:probe():210] {'os': 'Linux-3.10.0-1160.76.1.el7.x86_64-x86_64-with-glibc2.17', 'python': '3.9.16', 'heartbeatAt': '2023-10-23T06:10:31.385390', 'startedAt': '2023-10-23T06:10:30.007205', 'docker': None, 'cuda': None, 'args': ('--job_name', 'minigpt4_v2_test', '--cfg-path', 'train_configs/minigpt_v2_finetune.yaml'), 'state': 'running', 'program': '/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/train.py', 'codePath': 'train.py', 'git': {'remote': 'https://github.com/junchen14/MiniGPT-4_finetune', 'commit': '5c3ec8bb730b2e9df23b4bb3eb24108f9c04a8d6'}, 'email': 'jun.chen@kaust.edu.sa', 'root': '/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune', 'host': 'gpu201-16-l', 'username': 'chenj0g', 'executable': '/home/chenj0g/anaconda3/envs/eye/bin/python', 'cpu_count': 64, 'cpu_count_logical': 64, 'cpu_freq': {'current': 1996.2770000000016, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}], 'disk': {'total': 251.85773849487305, 'used': 7.905696868896484}, 'gpu': 'NVIDIA A100-SXM4-80GB', 'gpu_count': 2, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}], 'memory': {'total': 503.7154769897461}} -2023-10-23 09:10:31,404 INFO HandlerThread:54598 [system_monitor.py:probe():211] Finished collecting system info -2023-10-23 09:10:31,404 INFO HandlerThread:54598 [system_monitor.py:probe():214] Publishing system info -2023-10-23 09:10:31,404 DEBUG HandlerThread:54598 [system_info.py:_save_pip():51] Saving list of pip packages installed into the current environment -2023-10-23 09:10:31,405 DEBUG HandlerThread:54598 [system_info.py:_save_pip():67] Saving pip packages done -2023-10-23 09:10:31,405 DEBUG HandlerThread:54598 [system_info.py:_save_conda():74] Saving list of conda packages installed into the current environment -2023-10-23 09:10:32,054 INFO Thread-12 :54598 [dir_watcher.py:_on_file_created():278] file/dir created: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/conda-environment.yaml -2023-10-23 09:10:32,054 INFO Thread-12 :54598 [dir_watcher.py:_on_file_created():278] file/dir created: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:10:32,054 INFO Thread-12 :54598 [dir_watcher.py:_on_file_created():278] file/dir created: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/requirements.txt -2023-10-23 09:10:36,957 DEBUG HandlerThread:54598 [system_info.py:_save_conda():86] Saving conda packages done -2023-10-23 09:10:36,959 INFO HandlerThread:54598 [system_monitor.py:probe():216] Finished publishing system info -2023-10-23 09:10:36,965 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:10:36,965 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: keepalive -2023-10-23 09:10:36,965 DEBUG SenderThread:54598 [sender.py:send():375] send: files -2023-10-23 09:10:36,965 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-metadata.json with policy now -2023-10-23 09:10:36,972 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 09:10:36,972 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: stop_status -2023-10-23 09:10:37,056 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/conda-environment.yaml -2023-10-23 09:10:37,057 INFO Thread-12 :54598 [dir_watcher.py:_on_file_created():278] file/dir created: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-metadata.json -2023-10-23 09:10:37,315 DEBUG SenderThread:54598 [sender.py:send():375] send: telemetry -2023-10-23 09:10:37,316 DEBUG SenderThread:54598 [sender.py:send():375] send: telemetry -2023-10-23 09:10:38,057 INFO Thread-12 :54598 [dir_watcher.py:_on_file_created():278] file/dir created: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/output.log -2023-10-23 09:10:38,321 INFO wandb-upload_0:54598 [upload_job.py:push():137] Uploaded file /tmp/tmp2rfhjpecwandb/l4nz4hzg-wandb-metadata.json -2023-10-23 09:10:40,059 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/output.log -2023-10-23 09:10:41,298 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:10:42,060 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/output.log -2023-10-23 09:10:42,566 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:42,567 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:42,567 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:42,568 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:43,061 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:10:43,093 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:43,094 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:43,094 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:43,095 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:43,680 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:43,681 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:43,681 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:43,682 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:44,062 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/output.log -2023-10-23 09:10:44,062 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:10:44,147 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:44,147 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:44,147 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:44,148 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:44,652 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:44,652 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:44,653 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:44,653 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:45,063 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:10:45,121 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:45,121 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:45,121 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:45,122 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:45,631 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:45,632 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:45,632 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:45,634 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:46,063 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:10:46,151 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:46,152 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:46,152 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:46,153 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:46,700 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:46,701 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:46,701 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:46,701 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:10:46,702 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:47,064 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:10:47,182 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:47,183 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:47,183 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:47,184 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:47,656 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:47,657 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:47,657 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:47,659 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:48,065 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:10:48,206 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:48,207 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:48,207 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:48,208 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:48,676 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:48,677 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:48,677 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:48,678 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:49,066 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:10:49,129 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:49,130 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:49,130 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:49,131 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:49,579 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:49,580 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:49,580 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:49,581 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:50,067 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:10:50,138 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:50,138 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:50,139 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:50,140 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:50,686 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:50,687 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:50,687 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:50,688 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:51,068 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:10:51,112 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:51,112 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:51,113 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:51,114 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:51,581 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:51,581 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:51,582 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:51,582 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:52,069 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:10:52,089 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 09:10:52,089 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: stop_status -2023-10-23 09:10:52,093 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:52,349 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:52,349 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:10:52,350 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:52,352 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:52,532 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:52,532 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:52,532 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:52,533 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:53,007 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:53,007 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:53,007 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:53,008 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:53,069 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:10:53,555 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:53,556 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:53,556 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:53,557 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:54,001 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:54,001 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:54,002 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:54,002 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:54,070 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:10:54,511 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:54,511 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:54,511 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:54,513 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:55,015 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:55,015 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:55,016 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:55,016 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:55,071 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:10:55,457 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:55,458 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:55,458 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:55,459 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:55,937 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:55,938 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:55,938 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:55,939 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:56,072 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:10:56,406 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:56,407 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:56,407 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:56,408 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:56,859 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:56,860 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:56,860 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:56,862 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:57,073 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:10:57,356 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:57,357 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:57,357 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:57,357 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:10:57,358 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:57,910 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:57,910 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:57,911 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:57,911 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:58,074 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:10:58,390 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:58,391 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:58,391 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:58,392 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:58,848 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:58,848 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:58,849 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:58,849 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:59,074 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:10:59,420 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:59,420 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:59,421 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:59,422 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:59,871 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:59,872 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:59,872 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:59,872 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:00,075 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:00,395 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:00,396 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:00,396 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:00,397 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:00,900 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:00,900 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:00,901 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:00,901 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:01,076 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:01,379 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:01,379 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:01,379 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:01,380 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:01,838 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:01,838 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:01,839 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:01,840 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:02,076 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:02,317 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:02,318 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:02,318 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:02,319 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:02,825 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:02,826 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:02,829 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:11:03,034 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:03,034 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:03,077 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/config.yaml -2023-10-23 09:11:03,077 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:03,271 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:03,271 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:03,271 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:03,272 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:03,742 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:03,743 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:03,743 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:03,744 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:04,078 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:04,286 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:04,286 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:04,286 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:04,288 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:04,767 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:04,767 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:04,767 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:04,768 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:05,079 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:05,227 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:05,228 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:05,228 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:05,229 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:05,692 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:05,693 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:05,693 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:05,694 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:06,079 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:06,114 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:06,115 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:06,115 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:06,115 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:06,732 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:06,733 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:06,733 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:06,735 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:07,080 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:07,278 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 09:11:07,279 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: stop_status -2023-10-23 09:11:07,282 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:07,542 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:07,542 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:07,543 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:07,775 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:07,776 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:07,776 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:07,777 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:08,081 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:08,257 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:08,257 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:08,257 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:08,258 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:11:08,258 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:08,886 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:08,887 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:08,887 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:08,889 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:09,082 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:09,348 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:09,349 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:09,349 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:09,349 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:09,851 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:09,851 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:09,851 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:09,852 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:10,082 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/output.log -2023-10-23 09:11:10,083 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:10,319 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:10,320 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:10,320 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:10,321 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:10,819 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:10,819 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:10,819 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:10,820 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:11,083 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:11,292 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:11,293 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:11,293 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:11,295 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:11,762 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:11,762 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:11,763 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:11,763 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:12,084 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:12,244 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:12,244 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:12,244 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:12,245 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:12,727 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:12,728 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:12,728 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:12,729 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:13,085 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:13,204 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:13,204 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:13,205 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:13,211 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:13,762 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:13,763 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:13,763 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:13,764 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:11:13,765 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:14,085 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:14,249 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:14,250 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:14,250 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:14,251 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:14,757 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:14,758 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:14,758 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:14,759 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:15,086 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:15,207 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:15,208 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:15,208 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:15,209 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:15,653 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:15,654 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:15,654 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:15,655 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:16,087 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:16,163 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:16,164 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:16,164 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:16,166 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:16,632 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:16,632 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:16,632 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:16,633 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:17,088 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:17,105 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:17,105 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:17,106 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:17,106 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:17,650 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:17,651 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:17,651 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:17,651 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:18,090 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:18,137 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:18,138 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:18,138 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:18,139 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:18,650 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:18,651 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:18,651 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:18,653 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:19,092 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:19,127 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:19,128 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:19,128 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:19,128 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:11:19,129 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:19,578 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:19,579 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:19,579 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:19,580 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:20,093 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:20,176 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:20,177 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:20,177 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:20,178 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:20,608 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:20,608 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:20,608 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:20,611 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:21,093 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:21,169 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:21,170 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:21,170 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:21,172 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:21,748 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:21,749 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:21,749 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:21,749 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:22,094 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:22,295 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 09:11:22,295 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: stop_status -2023-10-23 09:11:22,298 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:22,562 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:22,562 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:22,563 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:22,769 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:22,770 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:22,770 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:22,771 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:23,095 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:23,289 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:23,289 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:23,289 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:23,291 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:23,846 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:23,847 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:23,847 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:23,848 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:24,096 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:24,310 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:24,311 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:24,311 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:24,311 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:11:24,312 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:24,832 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:24,833 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:24,833 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:24,834 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:25,096 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:25,284 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:25,285 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:25,285 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:25,286 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:25,790 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:25,790 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:25,790 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:25,792 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:26,097 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:26,280 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:26,280 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:26,280 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:26,281 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:26,747 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:26,748 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:26,748 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:26,748 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:27,098 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:27,260 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:27,261 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:27,261 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:27,262 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:27,733 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:27,733 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:27,734 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:27,734 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:28,098 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:28,180 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:28,181 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:28,181 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:28,183 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:28,641 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:28,642 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:28,642 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:28,643 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:29,028 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:29,029 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:29,029 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:29,029 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:29,099 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:29,495 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:29,496 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:29,496 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:29,496 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:11:29,497 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:30,045 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:30,046 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:30,046 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:30,047 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:30,100 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:30,508 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:30,509 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:30,509 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:30,510 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:30,990 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:30,991 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:30,991 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:30,992 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:31,101 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:31,371 DEBUG SystemMonitor:54598 [system_monitor.py:_start():159] Starting system metrics aggregation loop -2023-10-23 09:11:31,372 DEBUG SenderThread:54598 [sender.py:send():375] send: stats -2023-10-23 09:11:31,540 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:31,541 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:31,541 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:31,542 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:32,060 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:32,061 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:32,061 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:32,062 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:32,102 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:32,502 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:32,502 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:32,503 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:32,503 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:32,998 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:32,998 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:32,998 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:32,999 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:33,102 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:33,468 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:33,468 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:33,468 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:33,469 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:33,994 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:33,995 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:33,995 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:33,996 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:34,103 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/output.log -2023-10-23 09:11:34,103 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:34,465 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:34,465 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:34,465 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:34,466 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:35,003 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:35,004 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:35,004 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:35,004 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:11:35,005 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:35,104 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:35,469 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:35,470 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:35,470 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:35,471 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:35,929 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:35,930 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:35,930 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:35,931 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:36,105 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:36,383 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:36,383 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:36,383 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:36,384 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:36,899 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:36,899 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:36,899 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:36,900 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:37,105 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:37,294 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 09:11:37,294 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: stop_status -2023-10-23 09:11:37,495 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:37,584 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:37,585 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:37,585 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:37,873 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:37,874 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:37,874 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:37,875 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:38,109 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:38,402 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:38,403 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:38,403 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:38,405 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:38,928 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:38,929 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:38,929 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:38,930 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:39,110 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:39,516 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:39,516 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:39,517 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:39,517 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:39,995 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:39,996 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:39,996 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:39,996 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:40,110 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:40,447 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:40,448 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:40,448 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:40,448 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:11:40,449 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:41,075 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:41,076 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:41,076 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:41,077 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:41,111 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:41,528 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:41,529 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:41,529 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:41,529 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:42,011 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:42,012 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:42,012 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:42,013 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:42,112 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:42,564 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:42,565 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:42,565 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:42,566 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:43,031 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:43,032 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:43,032 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:43,033 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:43,112 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:43,488 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:43,489 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:43,489 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:43,490 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:43,935 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:43,935 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:43,935 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:43,936 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:44,113 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:44,437 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:44,437 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:44,438 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:44,438 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:44,889 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:44,889 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:44,889 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:44,890 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:45,114 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:45,355 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:45,356 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:45,356 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:45,356 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:45,816 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:45,816 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:45,817 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:45,817 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:11:45,817 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:46,115 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:46,301 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:46,301 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:46,302 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:46,302 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:46,773 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:46,774 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:46,774 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:46,775 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:47,116 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:47,298 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:47,299 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:47,299 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:47,300 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:47,772 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:47,772 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:47,773 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:47,773 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:48,116 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:48,217 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:48,218 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:48,218 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:48,219 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:48,690 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:48,691 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:48,691 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:48,693 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:49,117 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:49,135 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:49,136 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:49,136 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:49,136 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:49,674 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:49,675 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:49,675 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:49,676 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:50,118 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:50,118 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:50,119 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:50,119 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:50,120 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:50,630 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:50,631 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:50,631 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:50,631 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:51,119 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:51,156 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:51,156 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:51,156 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:51,157 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:11:51,157 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:51,623 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:51,624 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:51,624 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:51,625 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:52,054 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:52,054 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:52,055 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:52,055 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:52,121 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:52,294 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 09:11:52,294 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: stop_status -2023-10-23 09:11:52,598 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:52,599 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:52,599 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:52,600 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:53,122 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:53,166 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:53,166 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:53,166 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:53,167 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:53,687 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:53,687 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:53,688 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:53,688 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:54,122 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:54,143 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:54,144 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:54,144 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:54,145 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:54,601 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:54,601 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:54,602 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:54,602 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:55,032 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:55,032 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:55,032 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:55,035 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:55,123 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:55,591 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:55,592 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:55,592 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:55,593 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:56,049 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:56,049 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:56,050 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:56,050 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:56,124 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:56,553 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:56,554 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:56,554 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:56,554 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:11:56,555 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:57,051 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:57,052 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:57,052 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:57,053 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:57,125 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:57,478 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:57,478 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:57,479 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:57,479 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:57,909 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:57,910 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:57,910 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:57,911 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:58,126 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/output.log -2023-10-23 09:11:58,126 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:58,318 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:58,318 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:58,318 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:58,319 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:58,869 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:58,869 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:58,869 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:58,871 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:59,126 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:59,339 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:59,340 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:59,340 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:59,341 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:59,788 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:59,789 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:59,789 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:59,790 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:00,127 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:00,237 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:00,238 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:00,238 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:00,239 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:00,687 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:00,687 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:00,687 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:00,688 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:01,128 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:01,219 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:01,219 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:01,219 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:01,220 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:01,373 DEBUG SenderThread:54598 [sender.py:send():375] send: stats -2023-10-23 09:12:01,667 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:01,668 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:01,668 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:01,668 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:12:01,669 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:02,129 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:02,197 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:02,198 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:02,198 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:02,199 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:02,727 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:02,727 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:02,727 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:02,728 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:03,129 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:03,154 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:03,154 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:03,155 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:03,155 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:03,624 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:03,625 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:03,625 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:03,626 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:04,130 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:04,155 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:04,155 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:04,155 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:04,156 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:04,742 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:04,743 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:04,743 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:04,744 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:05,131 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:05,197 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:05,197 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:05,197 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:05,198 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:05,680 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:05,681 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:05,681 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:05,682 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:06,132 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:06,166 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:06,167 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:06,167 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:06,168 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:06,674 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:06,675 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:06,675 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:06,675 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:12:06,676 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:07,123 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:07,124 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:07,124 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:07,124 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:07,132 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:07,294 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 09:12:07,294 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: stop_status -2023-10-23 09:12:07,594 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:07,595 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:07,595 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:07,596 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:08,061 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:08,062 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:08,062 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:08,062 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:08,133 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:08,489 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:08,490 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:08,490 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:08,491 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:08,940 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:08,941 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:08,941 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:08,943 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:09,134 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:09,413 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:09,414 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:09,414 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:09,415 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:09,894 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:09,895 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:09,895 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:09,896 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:10,135 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:10,372 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:10,372 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:10,372 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:10,373 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:10,880 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:10,880 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:10,880 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:10,881 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:11,135 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:11,394 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:11,395 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:11,395 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:11,396 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:11,949 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:11,950 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:11,950 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:11,950 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:12:11,951 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:12,136 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:12,472 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:12,472 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:12,473 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:12,473 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:12,992 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:12,993 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:12,993 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:12,994 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:13,137 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:13,524 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:13,525 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:13,525 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:13,526 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:14,029 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:14,030 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:14,030 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:14,031 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:14,138 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:14,592 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:14,593 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:14,593 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:14,594 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:15,091 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:15,092 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:15,092 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:15,093 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:15,139 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:15,639 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:15,640 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:15,640 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:15,641 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:16,118 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:16,118 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:16,119 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:16,119 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:16,139 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:16,547 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:16,547 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:16,547 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:16,548 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:17,105 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:17,106 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:17,106 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:17,106 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:12:17,107 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:17,140 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:17,577 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:17,577 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:17,577 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:17,578 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:18,141 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:18,170 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:18,171 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:18,171 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:18,172 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:18,667 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:18,667 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:18,668 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:18,668 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:19,094 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:19,094 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:19,094 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:19,096 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:19,142 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:19,692 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:19,693 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:19,693 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:19,694 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:20,142 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:20,228 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:20,229 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:20,229 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:20,230 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:20,672 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:20,672 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:20,672 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:20,673 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:21,143 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:21,156 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:21,157 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:21,157 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:21,158 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:21,718 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:21,718 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:21,718 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:21,719 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:22,144 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:22,187 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:22,187 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:22,187 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:22,187 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:12:22,188 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:22,298 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 09:12:22,298 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: stop_status -2023-10-23 09:12:22,786 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:22,787 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:22,787 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:22,788 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:23,145 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:23,324 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:23,325 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:23,325 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:23,326 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:23,809 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:23,810 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:23,810 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:23,811 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:24,145 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/output.log -2023-10-23 09:12:24,145 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:24,276 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:24,277 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:24,277 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:24,278 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:24,797 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:24,797 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:24,797 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:24,798 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:25,146 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:25,336 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:25,337 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:25,337 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:25,337 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:25,813 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:25,814 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:25,814 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:25,814 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:26,147 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:26,328 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:26,329 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:26,329 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:26,330 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:26,770 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:26,771 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:26,771 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:26,772 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:27,148 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:27,263 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:27,264 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:27,264 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:27,264 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:12:27,265 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:27,653 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:27,653 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:27,653 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:27,654 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:28,148 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:28,219 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:28,220 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:28,220 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:28,221 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:28,685 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:28,686 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:28,686 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:28,687 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:29,145 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:29,146 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:29,146 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:29,149 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:29,149 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:29,703 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:29,704 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:29,704 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:29,705 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:30,149 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:30,184 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:30,184 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:30,184 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:30,185 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:30,729 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:30,730 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:30,730 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:30,731 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:31,150 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:31,236 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:31,237 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:31,237 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:31,240 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:31,374 DEBUG SenderThread:54598 [sender.py:send():375] send: stats -2023-10-23 09:12:31,738 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:31,738 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:31,739 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:31,739 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:32,151 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:32,248 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:32,248 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:32,248 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:32,249 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:32,654 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:32,655 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:32,655 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:32,655 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:12:32,656 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:33,108 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:33,108 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:33,109 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:33,111 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:33,152 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:33,666 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:33,667 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:33,667 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:33,668 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:34,152 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:34,210 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:34,211 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:34,211 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:34,212 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:34,654 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:34,654 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:34,655 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:34,655 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:35,101 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:35,102 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:35,102 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:35,102 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:35,153 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:35,651 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:35,651 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:35,652 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:35,652 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:36,098 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:36,098 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:36,098 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:36,099 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:36,154 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:36,644 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:36,645 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:36,645 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:36,646 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:37,153 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:37,154 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:37,154 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:37,155 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:37,155 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:37,296 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 09:12:37,296 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: stop_status -2023-10-23 09:12:37,583 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:37,583 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:37,583 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:37,584 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:38,155 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:38,180 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:38,180 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:38,181 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:38,181 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:12:38,181 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:38,727 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:38,728 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:38,728 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:38,729 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:39,156 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:39,226 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:39,227 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:39,227 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:39,229 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:39,735 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:39,736 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:39,736 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:39,737 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:40,157 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:40,216 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:40,216 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:40,216 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:40,219 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:40,690 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:40,690 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:40,691 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:40,691 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:41,158 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:41,261 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:41,261 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:41,261 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:41,262 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:41,715 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:41,716 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:41,716 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:41,716 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:42,159 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:42,285 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:42,286 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:42,286 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:42,287 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:42,766 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:42,767 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:42,767 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:42,768 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:43,159 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:43,212 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:43,213 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:43,213 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:43,213 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:12:43,214 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:43,666 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:43,666 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:43,667 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:43,667 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:44,163 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:44,226 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:44,227 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:44,227 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:44,228 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:44,732 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:44,732 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:44,732 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:44,733 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:45,159 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:45,159 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:45,160 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:45,161 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:45,163 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:45,663 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:45,664 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:45,664 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:45,665 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:46,115 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:46,115 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:46,116 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:46,117 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:46,164 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:46,562 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:46,563 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:46,563 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:46,564 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:47,113 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:47,113 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:47,113 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:47,117 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:47,165 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:47,557 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:47,557 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:47,558 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:47,558 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:48,122 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:48,122 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:48,122 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:48,123 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:48,166 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/output.log -2023-10-23 09:12:48,166 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:48,601 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:48,601 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:48,602 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:48,602 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:12:48,602 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:49,082 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:49,083 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:49,083 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:49,084 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:49,167 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:49,646 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:49,647 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:49,647 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:49,649 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:50,167 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:50,179 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:50,180 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:50,180 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:50,181 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:50,543 DEBUG SenderThread:54598 [sender.py:send():375] send: exit -2023-10-23 09:12:50,543 INFO SenderThread:54598 [sender.py:send_exit():598] handling exit code: 255 -2023-10-23 09:12:50,543 INFO SenderThread:54598 [sender.py:send_exit():600] handling runtime: 139 -2023-10-23 09:12:50,544 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:50,544 INFO SenderThread:54598 [sender.py:send_exit():606] send defer -2023-10-23 09:12:50,544 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:12:50,544 INFO HandlerThread:54598 [handler.py:handle_request_defer():170] handle defer: 0 -2023-10-23 09:12:50,545 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: defer -2023-10-23 09:12:50,545 INFO SenderThread:54598 [sender.py:send_request_defer():622] handle sender defer: 0 -2023-10-23 09:12:50,545 INFO SenderThread:54598 [sender.py:transition_state():626] send defer: 1 -2023-10-23 09:12:50,545 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:12:50,545 INFO HandlerThread:54598 [handler.py:handle_request_defer():170] handle defer: 1 -2023-10-23 09:12:50,545 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: defer -2023-10-23 09:12:50,545 INFO SenderThread:54598 [sender.py:send_request_defer():622] handle sender defer: 1 -2023-10-23 09:12:50,545 INFO SenderThread:54598 [sender.py:transition_state():626] send defer: 2 -2023-10-23 09:12:50,545 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:12:50,545 INFO HandlerThread:54598 [handler.py:handle_request_defer():170] handle defer: 2 -2023-10-23 09:12:50,545 INFO HandlerThread:54598 [system_monitor.py:finish():190] Stopping system monitor -2023-10-23 09:12:50,545 DEBUG SystemMonitor:54598 [system_monitor.py:_start():166] Finished system metrics aggregation loop -2023-10-23 09:12:50,545 DEBUG SystemMonitor:54598 [system_monitor.py:_start():170] Publishing last batch of metrics -2023-10-23 09:12:50,546 INFO HandlerThread:54598 [interfaces.py:finish():202] Joined cpu monitor -2023-10-23 09:12:50,547 INFO HandlerThread:54598 [interfaces.py:finish():202] Joined disk monitor -2023-10-23 09:12:50,658 INFO HandlerThread:54598 [interfaces.py:finish():202] Joined gpu monitor -2023-10-23 09:12:50,658 INFO HandlerThread:54598 [interfaces.py:finish():202] Joined memory monitor -2023-10-23 09:12:50,658 INFO HandlerThread:54598 [interfaces.py:finish():202] Joined network monitor -2023-10-23 09:12:50,659 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: defer -2023-10-23 09:12:50,659 INFO SenderThread:54598 [sender.py:send_request_defer():622] handle sender defer: 2 -2023-10-23 09:12:50,659 INFO SenderThread:54598 [sender.py:transition_state():626] send defer: 3 -2023-10-23 09:12:50,659 DEBUG SenderThread:54598 [sender.py:send():375] send: stats -2023-10-23 09:12:50,659 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:12:50,660 INFO HandlerThread:54598 [handler.py:handle_request_defer():170] handle defer: 3 -2023-10-23 09:12:50,660 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: defer -2023-10-23 09:12:50,660 INFO SenderThread:54598 [sender.py:send_request_defer():622] handle sender defer: 3 -2023-10-23 09:12:50,660 INFO SenderThread:54598 [sender.py:transition_state():626] send defer: 4 -2023-10-23 09:12:50,660 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:12:50,660 INFO HandlerThread:54598 [handler.py:handle_request_defer():170] handle defer: 4 -2023-10-23 09:12:50,660 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: defer -2023-10-23 09:12:50,660 INFO SenderThread:54598 [sender.py:send_request_defer():622] handle sender defer: 4 -2023-10-23 09:12:50,660 INFO SenderThread:54598 [sender.py:transition_state():626] send defer: 5 -2023-10-23 09:12:50,660 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:12:50,660 INFO HandlerThread:54598 [handler.py:handle_request_defer():170] handle defer: 5 -2023-10-23 09:12:50,660 DEBUG SenderThread:54598 [sender.py:send():375] send: summary -2023-10-23 09:12:50,661 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:50,661 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: defer -2023-10-23 09:12:50,661 INFO SenderThread:54598 [sender.py:send_request_defer():622] handle sender defer: 5 -2023-10-23 09:12:50,661 INFO SenderThread:54598 [sender.py:transition_state():626] send defer: 6 -2023-10-23 09:12:50,661 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:12:50,661 INFO HandlerThread:54598 [handler.py:handle_request_defer():170] handle defer: 6 -2023-10-23 09:12:50,661 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: defer -2023-10-23 09:12:50,661 INFO SenderThread:54598 [sender.py:send_request_defer():622] handle sender defer: 6 -2023-10-23 09:12:50,662 INFO SenderThread:54598 [sender.py:transition_state():626] send defer: 7 -2023-10-23 09:12:50,662 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:12:50,662 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:12:50,662 INFO HandlerThread:54598 [handler.py:handle_request_defer():170] handle defer: 7 -2023-10-23 09:12:50,662 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: defer -2023-10-23 09:12:50,662 INFO SenderThread:54598 [sender.py:send_request_defer():622] handle sender defer: 7 -2023-10-23 09:12:51,168 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:51,543 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: poll_exit -2023-10-23 09:12:52,169 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/output.log -2023-10-23 09:12:53,441 INFO SenderThread:54598 [sender.py:transition_state():626] send defer: 8 -2023-10-23 09:12:53,441 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: poll_exit -2023-10-23 09:12:53,441 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:12:53,441 INFO HandlerThread:54598 [handler.py:handle_request_defer():170] handle defer: 8 -2023-10-23 09:12:53,442 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: defer -2023-10-23 09:12:53,442 INFO SenderThread:54598 [sender.py:send_request_defer():622] handle sender defer: 8 -2023-10-23 09:12:53,450 INFO SenderThread:54598 [sender.py:transition_state():626] send defer: 9 -2023-10-23 09:12:53,450 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:12:53,450 DEBUG SenderThread:54598 [sender.py:send():375] send: artifact -2023-10-23 09:12:53,450 INFO HandlerThread:54598 [handler.py:handle_request_defer():170] handle defer: 9 -2023-10-23 09:12:54,170 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/output.log -2023-10-23 09:12:54,483 INFO SenderThread:54598 [sender.py:send_artifact():1474] sent artifact job-https___github.com_junchen14_MiniGPT-4_finetune_train.py - {'id': 'QXJ0aWZhY3Q6NjEzMjczODU5', 'digest': 'e1357229d7190882bf8b2dba86a37a56', 'state': 'COMMITTED', 'aliases': [{'artifactCollectionName': 'job-https___github.com_junchen14_MiniGPT-4_finetune_train.py', 'alias': 'latest'}, {'artifactCollectionName': 'job-https___github.com_junchen14_MiniGPT-4_finetune_train.py', 'alias': 'v0'}], 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjEwOTExOTI4MQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6NjEzMjczODU5', 'versionIndex': 0}}, 'version': 'v0'} -2023-10-23 09:12:54,483 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: defer -2023-10-23 09:12:54,483 INFO SenderThread:54598 [sender.py:send_request_defer():622] handle sender defer: 9 -2023-10-23 09:12:54,484 INFO SenderThread:54598 [dir_watcher.py:finish():365] shutting down directory watcher -2023-10-23 09:12:55,171 INFO SenderThread:54598 [dir_watcher.py:finish():395] scan: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files -2023-10-23 09:12:55,172 INFO SenderThread:54598 [dir_watcher.py:finish():409] scan save: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/config.yaml config.yaml -2023-10-23 09:12:55,172 INFO SenderThread:54598 [dir_watcher.py:finish():409] scan save: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/conda-environment.yaml conda-environment.yaml -2023-10-23 09:12:55,172 INFO SenderThread:54598 [dir_watcher.py:finish():409] scan save: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json wandb-summary.json -2023-10-23 09:12:55,177 INFO SenderThread:54598 [dir_watcher.py:finish():409] scan save: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/output.log output.log -2023-10-23 09:12:55,177 INFO SenderThread:54598 [dir_watcher.py:finish():409] scan save: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/requirements.txt requirements.txt -2023-10-23 09:12:55,180 INFO SenderThread:54598 [dir_watcher.py:finish():409] scan save: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-metadata.json wandb-metadata.json -2023-10-23 09:12:55,180 INFO SenderThread:54598 [sender.py:transition_state():626] send defer: 10 -2023-10-23 09:12:55,182 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:12:55,182 INFO HandlerThread:54598 [handler.py:handle_request_defer():170] handle defer: 10 -2023-10-23 09:12:55,185 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: defer -2023-10-23 09:12:55,185 WARNING StreamThr :54598 [internal.py:is_dead():414] Internal process exiting, parent pid 53639 disappeared -2023-10-23 09:12:55,185 INFO SenderThread:54598 [sender.py:send_request_defer():622] handle sender defer: 10 -2023-10-23 09:12:55,185 ERROR StreamThr :54598 [internal.py:wandb_internal():152] Internal process shutdown. -2023-10-23 09:12:55,185 INFO SenderThread:54598 [file_pusher.py:finish():168] shutting down file pusher -2023-10-23 09:12:55,185 INFO SenderThread:54598 [sender.py:finish():1550] shutting down sender -2023-10-23 09:12:55,186 INFO SenderThread:54598 [file_pusher.py:finish():168] shutting down file pusher -2023-10-23 09:12:55,186 INFO SenderThread:54598 [file_pusher.py:join():173] waiting for file pusher -2023-10-23 09:12:55,753 INFO wandb-upload_1:54598 [upload_job.py:push():137] Uploaded file /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/conda-environment.yaml -2023-10-23 09:12:55,986 INFO wandb-upload_0:54598 [upload_job.py:push():137] Uploaded file /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/config.yaml -2023-10-23 09:12:56,124 INFO wandb-upload_2:54598 [upload_job.py:push():137] Uploaded file /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:56,182 INFO HandlerThread:54598 [handler.py:finish():845] shutting down handler -2023-10-23 09:12:56,183 INFO WriterThread:54598 [datastore.py:close():298] close: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/run-25yuqsbo.wandb -2023-10-23 09:12:56,210 INFO wandb-upload_4:54598 [upload_job.py:push():137] Uploaded file /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/requirements.txt -2023-10-23 09:12:56,248 INFO wandb-upload_3:54598 [upload_job.py:push():137] Uploaded file /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/output.log -2023-10-23 09:12:56,448 INFO Thread-11 :54598 [sender.py:transition_state():626] send defer: 11 diff --git a/wandb/run-20231023_091030-25yuqsbo/logs/debug.log b/wandb/run-20231023_091030-25yuqsbo/logs/debug.log deleted file mode 100644 index e1f2ca1..0000000 --- a/wandb/run-20231023_091030-25yuqsbo/logs/debug.log +++ /dev/null @@ -1,26 +0,0 @@ -2023-10-23 09:10:30,032 INFO MainThread:53639 [wandb_setup.py:_flush():76] Configure stats pid to 53639 -2023-10-23 09:10:30,032 INFO MainThread:53639 [wandb_setup.py:_flush():76] Loading settings from /home/chenj0g/.config/wandb/settings -2023-10-23 09:10:30,032 INFO MainThread:53639 [wandb_setup.py:_flush():76] Loading settings from /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/settings -2023-10-23 09:10:30,032 INFO MainThread:53639 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} -2023-10-23 09:10:30,032 INFO MainThread:53639 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'train.py', 'program': '/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/train.py'} -2023-10-23 09:10:30,033 INFO MainThread:53639 [wandb_init.py:_log_setup():507] Logging user logs to /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/logs/debug.log -2023-10-23 09:10:30,033 INFO MainThread:53639 [wandb_init.py:_log_setup():508] Logging internal logs to /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/logs/debug-internal.log -2023-10-23 09:10:30,033 INFO MainThread:53639 [wandb_init.py:init():547] calling init triggers -2023-10-23 09:10:30,033 INFO MainThread:53639 [wandb_init.py:init():554] wandb.init called with sweep_config: {} -config: {} -2023-10-23 09:10:30,033 INFO MainThread:53639 [wandb_init.py:init():595] starting backend -2023-10-23 09:10:30,033 INFO MainThread:53639 [wandb_init.py:init():599] setting up manager -2023-10-23 09:10:30,034 INFO MainThread:53639 [backend.py:_multiprocessing_setup():106] multiprocessing start_methods=fork,spawn,forkserver, using: spawn -2023-10-23 09:10:30,036 INFO MainThread:53639 [wandb_init.py:init():605] backend started and connected -2023-10-23 09:10:30,038 INFO MainThread:53639 [wandb_init.py:init():695] updated telemetry -2023-10-23 09:10:30,456 INFO MainThread:53639 [wandb_init.py:init():732] communicating run to backend with 60.0 second timeout -2023-10-23 09:10:31,052 INFO MainThread:53639 [wandb_run.py:_on_init():2176] communicating current version -2023-10-23 09:10:31,359 INFO MainThread:53639 [wandb_run.py:_on_init():2185] got version response upgrade_message: "wandb version 0.15.12 is available! To upgrade, please run:\n $ pip install wandb --upgrade" - -2023-10-23 09:10:31,359 INFO MainThread:53639 [wandb_init.py:init():782] starting run threads in backend -2023-10-23 09:10:36,972 INFO MainThread:53639 [wandb_run.py:_console_start():2157] atexit reg -2023-10-23 09:10:36,972 INFO MainThread:53639 [wandb_run.py:_redirect():2012] redirect: SettingsConsole.WRAP_RAW -2023-10-23 09:10:36,972 INFO MainThread:53639 [wandb_run.py:_redirect():2077] Wrapping output streams. -2023-10-23 09:10:36,972 INFO MainThread:53639 [wandb_run.py:_redirect():2102] Redirects installed. -2023-10-23 09:10:36,972 INFO MainThread:53639 [wandb_init.py:init():824] run started, returning control to user process -2023-10-23 09:10:36,973 INFO MainThread:53639 [wandb_watch.py:watch():51] Watching diff --git a/wandb/run-20231023_091030-25yuqsbo/run-25yuqsbo.wandb b/wandb/run-20231023_091030-25yuqsbo/run-25yuqsbo.wandb deleted file mode 100644 index ef667cbe808972b307e42a84a435d31469b58808..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 85770 zcmc(|34Bylw!ZC2f@0Jbv7Mr=5p6|nDm4!{U9|`2c5H2@SV@YI7?O~LNo^Yu1jPXd z6vY8?LY%-60To3NK@>-DMn%OLwOepNzvn$kDz#7L9IuD^-|znJ^=fw2sr9b4_S$Q& z_3nN8pRn73?>)A`&<*bzn%KbVX^pb_l$8&ts;WvQLj!_U;hJbovbri2O;!!48W2iE z1_Tqq=yYp?CpRrE?HP`Z7&5rEs;RVM>}z4m>RB|(3jX@H`)oSACNwmY9uOa#YHqsF z`rPVzQrX6X>Kf~6o7;wl`)ppiS?PwA!_tj4Rimt;v;K8pua2>3<m-CUnt(M;X-SZp08cE{EiKK&3fzpEv~harCav2PP7&lS>uYWS;f{`{#jy`^%!W) zvaCKEMgploD8A>0!9XY)2!}U}1|p$A@W7(J)^zLS(u#p?ZOyIyD=KR1+6E4(3RE`@ zs;D2*SUoV)7>ZULfsB7U>bQQ<$^mtanYJN~ncjV4)sakfvZ^W(L2H>*&46&EDq2+) z$y9}-pO&hXoSifKI zO?$p&TYbjLk6tb=3xtE=NWW09UpP`8O!W^1`$yt`N+pBkCm(X;U;ln;`LS*3mbUV? zmNX`8tS$ZRKmEfwC%y9yKfO`??H0fNuKn9f@0>HrD!a` z#((?@#KW7?HAoW*hZ3RkmP~b1OHFI}0OYh)4%~*iw(OJVc=bT@2vI4JG3zNz3rR@ z!rR-xyJGolpLmbl@BX#IyQP75#S4!X6t92#o-%uvslxkv1MjNUkNd-6`y$T9X(-@a}Hlo$;q7POop9cdzVJp1K7WtB|3`K6$E{oA+KCacd8-k^bZ+9xB&`rGf{_4QRJ2ye*1 zyY!Cpc;0WXZ{*syF!T41k!ubY-mrmp;ZLJ|;tg)HwpDl|2Hr{Mk1Z%(|MtE5!j^>a zMh(34o*m~C?;8~-*!%sMfp_FD<9*?c6@31Fb@R0q!W%d6j{N>&pLn;a-0@1;H(}tN za`}XU;`MLe9}hax#+x+oPWtdNpLlO5FIpzNDFg57iC3;S-onh^`_I{Hwd~v9z&rB2 zi9Yf6pZmq*!n?PDcjTK_7Zk65`_8+!X*1#7$G|)7hHHG{J?+dxCkyYs2HtVsUgryM zyx{Zq-LD!q65jm`ydxLf;1h5CQO7P2-u(@{vzARMC|>{eJ!He$-wW^m7o$3?s zkN-Z+&Lk0WH?gABa$KAKigy#DRm zr`NsP3Gbf`yh~?J_ldXk#qFjG?_UhOtFFJt7v4m{=kL#-?{bLn{?))cW8@5X5d|V&3y&M>)*cbXLd^q@4*J%6)W%eiFeQ~vzmqX5CiY5i4Uze-onh^=0US| z7v4h+ybD)6>=W;jr#G?tDTf(&S5KK;P`v)_`@0Rd+EI89H}Ed~agI;CeK-7RPucef z1Mk9xkNd)#EcpE0X5ueBh4)AU@4U%R_{6)(s_Tvr-oG1or(Hk4pm_b;ci%U*I8b4b~wKr}N-eU~B<33;L6YmkXZEg1#jy3SE z{$f!<@%p##nfr|&BfQ5Mcqh$#&L`d_8y~lY?0dX{cixvT_`;hi`277g(eiKM{fB{f z=_4=t#Jl_SVG?>`N^;~rnO z-gpZ$e|NmG#E#D=8F)ut{<=@RHtTll9Q`)(<~ryGEm{_vqsz;A5w%_sp*8-N!sTDAUw3p0hEK5pt>0$gPP zo_GDHJ^|0#@S^txxY_``^ux~!3)sJfUvC<;PJn9+z$<2a=@amh2VOWrfHMZ*Ae0Iyp3txv$;J-hHA0j@Ow&wA;4ooF3cQ$=gZ`! z0^DK%9yevYPr$35*ljZbZZ!apyY}M30`_m=xj!8Ev;emmfM>jYiBG^sjC^m501q($ zPn&+ZKfr}2V^#ldl$}}*H2_apcZE;DyA|*Ar~nT$0Ixp(s=@;HZ{ZyhRj0_p!wtZb z7G3QVaLeP9&k^7e2H<%YUc3H)3p0nm{Qm5T0(_4r^#?eYuv3@=NuN}nUnaok7=TyZ zcZ*NJ%RYT$jR61K0KD{p+X@TVzlHB>7(Vk*1RQA@h)3Qy%_rj9e?R}V$vp5}%YeN0 z%jxS6xiGW%xu?mILiTUvLr)oTmmrTeA+Ne* zhEK?!fBE#!f;`5AJn7r}{Xs4~HEYBAoi-BW^G(Rp9(&LyW;3S<@;DRnl<|-Fggky_{RM)2feCrV{5k6nxiE8iqx}beBFGn-kXK*ws87gm zKHj=gkjI;lSC4zVu#o**`OZt0OcvyeOvoc&ndcMosppjIONZ|x z$d{Rrr@XXy{UH}-E{_;o6c^;nO~^B@U*Z$;7W)o7T#&CYA&L`Gnea+x%&V?zSe}i z;*Ga`LO!?up)U*aWE1j?mG2f7vVSYDoc83If_$9`dEUM6`Gg!ff3F_}`Fa!b^2s0i zgIsv}*64pto-W8Yn2=|E_mNM?U){6sJ%W6r33=SePYMg!zm;#e2WQr!bRWsgdG29;jV&ws{`^Q|MdsC@KmnEk!Obm`8E^sii_6xguKtuTmMIp zZ#N;YIRE>?LiTUvU7tB=cR`+JLLRs12cM9y`)SYL3-TQ%!v*E@b~!-rQv?uSmbD|*M=Yag_(rEryre}BFJ-0$ScO*R9MLVt$gU!C3_0;TnFSa zH~WMSczIm1)KW0K+b^SD-kY~R){cS;h+=RUJ#_5HH z?BB`{?Do6Q1bLnbdF}gm`h?v4-8Y*E@)IWHRpakjf5?TI%S%Q#4-n)hO~?~oxYsA- zAuk;>Qjq7HkS9Dlv#^l;TY2Yi8|Mr1QzqnD7v1j@@*5u=_@*E~?SMS{A%Bny&-N;c z@32OY7dRlVd)O!B!&d%aZ!4ZLA6SMv#}9kY_!y*eB%k@94FIto))0dD^E-)*o_V=JG519J-?*zhpvQdiM)H zA;0;>pLZ7Imrcm4ZhWz@ko{YE;$gEF3Gyo@xG5v-^$1Rc=G9jyxfF5?xr_=LSBCQ zzP||a>kh~-zP?@1A+P=B`@%x@Z{?*sjND(4SDBE< zJ@%td$kTWJa)BU!;(+|oy7h-#n7KS`ufuI}@~0-`Dbs)P3Hjf@T(yTFe`Z3Sa?80F z`spu+a~(SuvVSXAT=n4IvhwF9#%>+=bPh3wzTWByvdR*?T= zLSAw2=}k0#_*cidE1$o{R|aP%HK3Gz=SA+LS+j>1CrZ{=t2on#+07+GYTU!HRF zojxIlj_+@uc|5nsJiq+%-RlpzFmw6v!)Dv3;YOK|$K80ZPsoc;d;0-d`8*Tyl*?uo z7P5aUe}DU7{p8rAO~?yhxz8u$hi|>q=6#KEKpypwKgfmWeI5GnR`#~ydGe$kWC>QCP_St$bLjZ;2pZWI|pw|4E;ae=J*k zXqpFJTvWej&Mlj3SNy~Sw>56rwBK3m_hrS~?z3}~1i!=-eA?oleG0zo7n3hMZiAwp zmloCc%@KUUl9A)BvaRz4uc}M`Z{K@ew&U{cHrb%0w6te9He$%&)~cqD7VWaNq@Q3) zKW@q>e#}pq;=PqSesU=fysW7H#O&*|*UlZs;|}ijZQ`j==L^#thYYI9w3IguXn%cp zx{9}-H>X?DgEDQrXS#oREFKRh!oi@AH(qc0{D|qgaOLu%`s%DyGhV(AsgCVls&J=N z<~!ueGtEuBY&`g<@=!1sEU%Mf<;_T4*VxEE?NL83I6rFMNBbXYaVWW{{_-Lue6RgY z^Y`7brl{=PqKbh{gEAG>12c{F!P<&+W15%6*Q6sAnZ}{56`2v4iiWzXislh*1DhHn zfmB5+V)biISD%rt&9qjuHMLX^tQgW(*U+j*RG-n@RM*%RXdcl&66#-md~2rVgmg<| zU1M$k@*|rXGq(PDTYFt&^N_aI@`3508D8x^ct~AKrnR!RC0(=caV=im%e@0|CxRE@L}<%AKdNPQCR%SqTc^1-K4aKZJAM)6)fGp zw4|Y_l_}FZ6o`i6p+qbkjfW%2L@*gE-MO^1vaN1VrnN0Ss97J0r;@>FC>jk1V)0l! znu?X~RNAw$Wk{nwr4KL=PK9E@Xfl>cg`<(Meyy^#EyLHXVDAkEoOZ(a`3N$xsD8X< z&*%4T+pb-6(IrJ?BaUlH*ERMpuSvJ1+r6iM`N_dk%gawLFE0;P$jSJROk28td9Z&h z+CLI0FK=k!KLU&dgCYJ4Ed9%EDB<#QN!7nR5(pNjhu%llZtwuV~SGN=`LG@ z0+A#tkA{+=P{i9?B7t}!6$^#Ckt80C1o?U>mWW1!Aw!Zdk`!$`d(1JCN`?g~?j;GLCQ`wa(>&2YJfumKj4%rmvA7{h6j4eJdG5&TB+6tDQG$U~ zJQ>F>(O57NN<@+#o)Qa$LaA^v8Fq>i3j~wlL=suTtTw55G*>GzMCtK;^qw~*%5@&1 zgab^l1ad_2wMZ-$^{`JY5RU|d%-4KTVthA@IKf0X6ph4lMTsNIhI{v^dO@yoeV(i2 zI&CZ*O9d0b2w=jINYvdvaXclJlruR(90Q?9Bt(=<#)F|?!ca;ANjBQ{%QGiSk{j|Q z$#vRTDj5&QFmosvi^h{lFHvyGWCTxeIBh(DSt5}{1QRm{j9{BYlAb%?xBU~68MjemUdln>8Gk|{;KM_L-~ zehG?)q*zQHOeEu|*-Mmo4A+h%o%TsEf0D6CG|6Bp$VW;y%`QPjp&-JPKJv^VQ{}Zc zd3bFwkc@^>7#U}?O_V4tHWvaJc62Vv^5e?!dk#HoH@K~CY1lwqgh1{tZ28h@sjiFc!he^c^VH9mQZQAF~ zt0m0M-ol^|&;+9_L17dfb~jQgK-5n$>z%@+Se%$&!El^RBM}Bru4WW)H!CaNc~1#* zi?=YiEonwDo=PN=Bqkm+Iu$^NsT4VhV@5MTQj?0ZECh*yteCmNDCGX`f>puk66RKK zVaSQX(O@`4VxDAnySr~J7zi<;QGi1jPPztDXe@*eC*tThX$YgByLr{D7sGPj+dSNt z6e!N_Aclz$giOpm=pqX%XClZF zRwU9_TQRP}kju)Rc^eTlL;jN@NJYaTTn3NInbTo~-mR)`Iqf3(?p=Alo4ZUCbYleZ zWRmrS4M^N|6^KOw z;RHD*tE|##w*?7NQmGUxl5;}yck(z^D$I;9VM8g3y4%e9{})SbjTcOohBZeOutVq6pHKF^7SVhIieP4piRvX^k3)bT)+p9}F3$9fbG z5So}wiZbLK=A2gC-C^=$iCz-r{yb5-;;xoPcCKvENT*Uk;=6|=timKJQK$3914(uv zp%}}3BpN4p8d_1{{X^s5``anQ170Q~)#UdOHxeJpB;8S*xcg!RPWLzRb z)LyWRX&q7E-Ld42z86R@59aA5H^4*zk|1-Akgdi^G`(a|M$1Q?CQ1a7BtLSc5Xq*w zAVm~*cN#M5)R_|HAx}}**07f+V~sGMqQR(ptV>deAdd<;O~l{X?1e+gR6If|W`^B} z!fyHV?_BkwM0wa#6e<*2MWXCXL*Y1a%GF(x>={`2h(HdedU zdGue;eO{u>@)kw4s#KT^k>JZ}>n=))7((JlmhTWHg}Nw!un{0Jj3kU)H=?N9r}eD) zc6s3uPf^Ib?PVmzA{!2R>_}p%K$2>c)~h^GsBy5FW@#j;BPi!YmWX2RF3lHQGf(T{ z?7Vd`cRGie7vu?q6$&?OeBA644+aRPtT@hf5t;!WOWB1ixIJ&rl|%t|*D;5`_mL!- zlP5{n1R{${VSWL^RZ>xEJ8mm9q9mE`q>iRekc53NMLVKxIE0Q3ohaJ=aZ$}5|1M9N z>tPOR6jU~dsNubCsfKyW8sH&)HC`(4*cV zj)|NCN)SOpR*DQ%G2esf9H-U!ayWUrjlf*7{}H|E*Dj--ShS5_rF2D@`SgqkU_KX6R}Y+ zx^WM-@kl@gkvLmHb2&tq1Wk%~JWAppWBD+AC8}t<*Fp8Wf2@AUlX?A+T)nXJP;tQ_ zgDIZqp83b4fv}22c?RO|QS`y;gug`y2f41ISo^1qzuo8^8EAf{Y5%Ni9AiGyX62;oU^yd!zizInEPm)~gFmc$x$EdYOsFJ(qO>t_< zBrA!$TqVK)x=JX+i;;zyk|@?zJhbi5f66=yJj_E1N2M|vrY=XAq<-WP-&8wBDWv=k zrNr6uk`z-3K+-UckX%nutPL)?Z{^(*$Q)9uKfc(X7a`4@Kov zRJlQv<_tzQL{X#-U3$ru$4Hcgo}wTE`(c%GahE8oziU`YFnvNn>Ud7AFrd;1xju1{ z3P`RfF@@Ujg0>@~^AY9QY(McwyPr7W!_mA3&8wdnbM_O1$^KDw5WDXu(pl>4CkCl= z#OXY$OW9pNk=<7ExAzlwy?O`xT?~sl`-$y9rrdsk&-b|vA2GX8R$A<7C88`n9`X%R zIGoqRcM}0B&_qOM>ny?m$p)P>lDtsNtP;kw3`Mqid*r3(BFb|`z0Wpl|J!x;6MIqm z@Ud+`(4qN=}dY} z`bFuJR-vl;u1Oqey%PRd=^t{NiM=A)2e8j2Sz(LBx}7tbV_JV=+su3NW<|K?^CDcA zAY_vYZIrjPpQ7>Qu>g_ON8>D+PL=R?x-}GnnOt;Jj0_>B1t{L~ulUvSk{3L@1WA;h zhgI>UsS*i$tjZ+&+C|3l)*p&bl_TB-kbsZz-bDdkYzLgqYT#)PCnJu#@bUyoI4_7AHi;$ZIJmu*Go; zw~0_7tj#+U#7vx`46q)?v`r;LP7#lD9apPR{}Y#eV7DG#_HY-<<4j`~#U#xKx-yg* zJ!H|cr|qM|NFnkSLJCum#hpEsA&OR^z4!m(Z}sxrSMo&3br<>;LE=_4#nK#0y63hD z(t0XN`TcmfE))dS11FWEIbmugu2pEC&EMbr5&6rj-X7^^Sgy@mtFZ4Fqyeryyn26jy7)BK5 zMkpE~VnZLHfJq-kJ>MK}i7Vdjx8V_ko|cK0d5WS^9CMjfUW61@c~`ZN*<%8*JH3Sg zRZ9~rA*v`C5jU=gyZ_WBo4qbkmgk9*>n%(mvXvNFKl8w2cbKp8pZ!N^WwiD3JC%vH99*?0rhoUy+!Fgl__>0=JFh6fW=uQdbOraXHm#K=+9f1 z>?)7rfU}Qrn3b=p%?A=3b2FKf; zN}+;G<01-Sh~-i1vs=oNAiJlAhy@+-HXa~TM@IA|Dv&vu6>tCY;M`Xpl~UfxQ%bIR z*t|sKtrU?7Q&4?eC4n-+zLsvVV--rUpJEAvghi)-h1GD{gyQXAx7|H(qC|PuQxx_t z6dqZ46HsJWk=zXgWh6@9D6eiC6_*petKJVn7%*-}BR zq}@Xq%fo4*jNq1ZZJY*5iWfkUfgG7!gG4wHw*++JvI*A#;zm)=8B2^F+y=K&ScR@ zlqNVkWl?A(*%o9_N;b5jXnW{cuYX;kX#0V8v?b`1XOnR-gQD=Uyo8}tOc}BpVPZ5Y zV4=W8w2L){Fp9Q^y|wx0^CiqmZ(%4$&~>Ii%Ss}$nX52KXe2a!Fr;G^+58e3To@bP z9R)tqNQ$?IA3nKmfa2|kd7;HH63S3w6hb@9X)jSQ5fjZ>(nge+X0uXpc88=Ixt^>A%&=bk-6l#SiMBk>Y8-;7&x|Y(dFgbTDp9ok z`y21CJy|MQm8X(iG&532j-HIH0mQqsQfL8&X?hY2L03LD)gDi3uhlRVFDc<_e-vd-O{WJ^qta z@@bw*a=nD`3ME0wF*|T7mF|*|*it^vE0sc?kQCHPS%t_bjioTDNPEnJuWq+HWS`}Q zw_HiY8zhV5M@X{OUvZ0X$q1Wf8n1a3ene3!1=j**Gcns#iX!c?kEO5sQIYoZJW+D3 zLkSI@DZ6*IguLQOl(qzQ)w~KnG!9OiF!SX1s`Q(pDAFGH(=*S$t>E^BcW|SB04BD4 zDm}?Su4;+W4H7@Z$<{y^B5WLn1k;?AnNXbTEQ++pkC<^_x%Bd7o?dcI#Oxzm!PaER zs)M=9LSBnsV+&_YVE|F=tcwjBsaLKjinIUNe{5oj{PrtPzYW98Bz#6FyoT9ushw>c zzUYc6F=_@_z!~xRGyV<=txfFjO$#Z`o-j93vbmJ9`d5^~kqC4`Gn56$W6zw7;}n=E zud*IvKnDZZ2&kZ7?#Ge}v?qSLWv`;*OcvB1%GK z5td1yKvy;FqfqSm8QrC57P z^WBXfN-O{M6h#CWl1QEZpv~nGYQ=&Ud&s$Qk1-%NIkpvD1dlPaqF8(Cg)2ULO)KGd zo+}}HT$(`?K}i90`o=X&A$dhyT*+acBs2oC8ZoZDei(u%(w=t3);AojNV~=>(o%DT z+e9xNhoRX|{aJI_E2sKdrCis~h7CsQkmw3Sb9R2!EtYrTaLr$NOpQWdll zb>DZSU_guH!WPKf8N!)RXA9dyd zYPS^#QP7;`nxlk(C`3o3ItN3J zLH4(SVCP`iYcS#%gjbykm6f&sX4D)ki8CoW$6}5#sj+h`?iho8l>J`9F(x`t=l7D1 zG4h>Q=UB=yrmhNOy``IwLpf+MTDC*Sm{T{DZ$n9R)Ba6|a!N?{64|dhl|#`goXURH zsht!NJ^NLMdV0Wxj>uf?zv@&F8DAtj=+sb@a#42BsUpf~;p~q(bku_bb*V(Bl5iNq zgNoZ-#1;sv8)GGmMY3OX=*ak@Lq&YicG>o)IyJ;oyToy5h^J=l8FuK$7;~t{kjAMU z8%BrH!w&5rDoXG0AShpk0&<@U@5vZzS5Aj}MI6fMK`4ksJIw_R#{8-%2H8QUdSLlx z4Gk~C{3hh9Sf`;Q4*m29ak?{^9d&4^2lU}kIQy&J=qPKHh(kxV8ah-&o6L|XS4DKK zj4|mpi<1fG9K^XgtU?2$o59YXbZCg53}wIQR8X$1nKI6&hPr;zp`UhpcFN;a5B<2V zZ#vY&H^W(&lvtd<*=1{r4S9pv$Fl9t8_b3owx4-}S$nYwGX`@N6w6momk!9tjW0?E zT}Zl7P}iVSLAiP-(0B9L4r?+qj6sCWdL^VoV@&eII!z7tuA9Nmr#kdwyX7{ey|_W6 zT3-va4Q$DzYdBnA6xO`lv@z2*tf}RUK&!ouma3%5nT2*kW(Qj`)k8;FrEJ)P)c(lf z%l=rpjr~ZDt2ecb2&9Lm>l$<)#1RLMvNop5&iRT!C|1_1bi?*futc*g5q|$rx<&36 zTUuN7m5u0pq}V|-X}8QB&MxpQ{?TaJPNiGb+KYetLl0fAZN_f`kwCZ(%+wcd+9dp+LtUWBSIHrKS2tFwmCK?FS{$I_&6bJ|=hY_wC;Z zb$p`-u?h2SU)rnvIn7PObiG7d8y>MC)oR7uT}rn#pV!@oI-U>;n1f+@0(Qob^Z28( zSsFWS?qDvTa*|!rZ|lb`B71Kr}CsYa!9|IRz|*+pbD$hPCcKZ zZNjvYN*S3iSN(Ldy!YHfbSe-rhLawA+j*Zm;-jJPTl&U4=IqS+!-MTAKK z)dXCoB&uZ8E_mas!BWb3c}mHx?{F)DxLPR+&?@S?Rx((A_z;X0=Zxm>)W6yB>qro# z92B$9oR9W$7D#zIwPIIP=!%JQ=c*Xa72*uETb<6r07n_@t0+_%iI~Eu zmQi=?^Q~V>FJtobl3O*_v27i$hHFTap_lGf;z@Ddki#)4XSW@iHZH>Aue(n$k|VmP zl2O0&HZPTG5}og@7b@5~+()H|vWRZzaMcT^3aOThh2RKv45)@nKa5eDTBcr9%{b$l z(TB$5xnuJ@Huw*86x!j#*=|P^E*jB!U33CL!}MGGLW{=tc3ru%M7c0elw7}s z9ZMZtF^(QL>z=z-5~>F#@<^O<{!a0k6_eRakIi(Ikm7AqI6C~Kk%%(Bxc6Bm*Ed%y zx*v-3QK|>hg3WHc(SKPe}j3Wbhyd=K?uP&4mCo@REj~gcNI=cUbpxwKQ^3 zoC!wS<+?38p^>VFIRs)+ zMO_s`t&b`-Tmz@SFhKd4g&Y1LE@z6OaNBm~)^ARhUM6_!h2X@=Ix<6U0+7$R$wI!u z9*Pi8f7A3BxWG8R`asmM2~CbUq;Nar?VS%DDq$}37KWA3UWj0S5n4#7T@*tO%SM3( zg8IHg7%i;O7_>H%1+!2Y?yHD9^sQ+9#}ejpZ(&$-2`vDk(`Li+5GKYtoZqU0n5-LN z2vJBw-r#Uw#oS>dXI-&G9&?4K$B+Q&*kcgF6-V0LiSz6?xDbF<*co#Xg;>m7q~s1; z7q()qUhKOph9A56{hLXYEAvFjjWDEPn$YxX=^sMxcGZfm)T8GCnZ#itU8x6Kh4wRR zx329mL{ZEgG4#5|t!1K#UMA8lFPy7T%be&BZ`ZX20~LzhPfD>dUl?W>=zFG z%aIahl9wjFFp9kY8Zdt1l@jI} zFJah=&?zOMV=KW?7@fLxbzh1ZWcfO%@6ZebI_if7b+X5?FA6L6o-=W8>l6udt+z1T zF2-8VaZHvJ{j0k$F^+R{Q$)V|G5}E)76+|jB7>PYgcW=LefPYIIb#uKa&hl-JuWKa zgar&@&bN`9u~GBz8IoGG2aUz483-G!upu^{I%enC1cZ~li+hf=KApa1g`~O8TN>Tc zg!k$iMsDhZeBf#)m>JMW^N$Ka3}#Ow!V~U`R1Vdokmg)#-J0uvku=x;iZq09F~L}q zm=4f)JfzWiEtU+YZge`4xJzhY7pt8@?tJ1(i{hT6toNRJ<4Z|%gSRwrKS&Z4%{U6; zac2Tvif#zkdEB%F101QLHbo34&ogH?w^|hUJkLr*pZ>k1xzSr1b*|as(CVemF=B-WQbx;#xPYtb{%*)hG7->dZmFFy4C~2m6OT%O$_@WuAFysdw^IM1ObWSHv z8k^f;U!H*Psc2_dDuOg)tV_S|Gepwd?dF>sqPiW-rxHD}zL(vXBnFLF@Hcx@iUDTZrRS4FjC2Cwa9`RXJ)g zkvP(fv&tUqzpqqtiZKYIHL@cZMC{g?;FPLd?SwEG39f}xHAo`@EZHC#50l(nFe8e@ z7h0=N8fJUVZNH)#?Rm5*W$o2z40mZ@SWpkpVF1%>#9oGs%Wjk6nJNaxgi;_LZ_V6) zlWnA$+r3r8nKjr<#QP}8jV`it3&bHP6r`z;!W>?MFa#@>FLt>^b!dsX(nJ-AFS5oj zT6T=2ndT*pPC8Q9(UlA+hal*l^KpwbsXL!>rk~K5aXYpM_7N86o-(RHe6h9ZlsD`d zb%&QUq&6&daENr}1$0MuY9Xr!-T{R--))q#6RB9jDK+7Hrfw97FR^ZFn7vH8neHu( zP3F_F8J6E9odC_!pt>zI7ArZ4Z*71Mpfq*juaa4i$Qmd|H^b3;aPA_Sg-|UfD z6yQD+D+o7U(r_UcEl6h|MjA-5y4i~4278$s%cuhJ1Z&^w6%C3ucX>@IrRPdZ$aPs= z?DOZY(MGpBLn?M=6G>g}Ofte&j!UJ)e>R+0f%r13G~B#cs=3=sHEN796^J(+;S>Sc z!%$p&r%nM(H!~wf8YU%dFV2Xw!m}qdq){Ng+-msSobRQYd-7G2ySrou#mt6^MzM|q zx9)bL5N$8<-Kd5QU7U1+B~zD2n8GL!Ut#Ui^N+<+&Ar~LA-Re%mzWJ?S)@x|($KV~ zFrF8PV_cz#zi^EJ8semeDUAa0mDWGk9kNuanc<}xR!kklVHxC?yv~WU$U`@zM$2hW zhn*N;Hp0p#L&C=$!YB|=wEo(2&Bap9OmEdtF(qxJDu*;$g56caIXsqn=+61l#As*| z#^H0?#oJu3i760YW$kp@vm@st&3(n4x9i_!cO=)If9*KW+x1!d@>`Ojs9Q@?*2`^4 zruZ+H+b~os@NpZ)LkBE;T(|4rU)=G^s&*jLR^0#)Key{wRn7f_Iv5Y+bue=06N?Tl z5IPukV&a}nJjSS_{rW;=pQcroE)Ay}-OCD749 zhoKlyYEEO5y($a15rkt}pRThuoxfs=R5ROKHQ0vip1K59VpZ@k6oe#B;Ib0sOQREV z?QK#OOH66BKwWQrcGpuoOPV=e(ulN-k+^sYshAHQ^-uV!c4vH_A|R^9_~T)?gBl1%nyivDP9phS` zrdanZ{P^@RJoLwk``;xV`b>Lunl=7?FCKaJAPc{5N)(;(UpQ z=iu;}gANW~j}Re4+(7%A4vu3F_5vX>AnidX$q}ln_+j>#gX7o(k|C73jxRbnj#LP< z4|P%;(YKhw_CuWvNA3m85;>hb^b%WAHzGQexSX##^aK45c5>%eo%(@R7S2+IVR@VM zNWR*|VRtB~M-))+Yp?6R@ zoh)>l%Go*Q(2p_ZP!D5SnsC^uoLstU*r6O)r+8K;U)-r1`pc5>7m`Vlb|)???WLqnQh;ZSzep(6WCL6({8 zB%+s)d3C{Y=%|ERd)J^tNha2AwbY^BZh7dPV-6)@mT;%%Icey*UeX;4Jxj7?#yDL) zYfnyu7-I&EZdmADBx|m0G$r{ASmr?S_Tk_1NxM=vja7j)k5z zS9dJ*u0e-_+Fi55qNu75m??(Az)g!9vHyQt`45HhCwHy`Ax$M0^(#x(MOP&Tr*# z&>`1{og8#n&-O&=mV@3t*t?yz-oZgHVTTee>mZbOQqX%aIf-LLPm|N#&Oz@O?Vzo9 zlF&;)ue79F4m$KI&P@FZ2fh76-E+_>b#tz{Q=(uyLA{GVu6wdlzsf;x{}hw9I}UmY z)%oO=`^@}{ve?J-v)J5}8lD}O{jh%$1C~Cf>rRa29{LX3>$s!%jwy7vEi6P)g{V>I zZc^jgz~5xeZa!{-b{q4&b{i_<>jp?Y4MHzfI(K2nZA8Dx+lj$@)_s25JfwT@xp%>o zMjQC4)_$ei+Z=}{yj4R@jmjmD;h?C_1iQ9uI4ubIj5eh+!PnJukVeGi(gk)7)oA~I zvvu_!$3806Jn5|(`aawhN|i+~PoP=lt{NR|VV9J5Nrz6npc`0jcBcT<8HUpS{TA!d zA)kzrH1oZsVM_=rK+S5dl7O!4B@M%DW}Ve;q=5&(`F6_8?Oh*38tvb2wHnqwWi!m4 z@-`HW2Fev;3~}0;w^O)FLsgqiC{%B!4>KT^j~HgOA04XE{{1#9ea}5Hspe^K)xgc6 zBmw0`cYmlT>DE9Z*QNM@J@R(U4A?K>5r0I*Vbf6Bzu#_^eb;w{q*?GQ(&!~C)m{K#0+35%_m6doSQbK(f)m!HDkX^?ArS?-i8u~gu*erbQRffV%PmU zXPmgGJ&#rZJDi)9s0l-Cfl5V_)bJYZ-|w(a{`V#4Ni_?9MKvf{6jnL1Zu4d#qD_-2876W=XyoQSMi$-wONrTOCk8~C zU^W+0`3JsmPhz{29N zlcMlU>+$)G+ljcj)PuMw(gI5k>kEAewRYU29v43A>_pyd;_o6Q=y(#^H~@LVxA~nC#CQh3ZE3yiRP4ZU1ySB?t<1t$~5m2QH_wLO&tk3bKFGbFp&cA z{nk~doUzk5M0u&WH>Yx4UtEe|@HnzYs06G~?g3bDc%g$w6S*64^cD$9-DD&%L(HBz zm$4Q1e89SEU22uYdHGkwVL72~N$jM558uLFJL<2ql<^U#cGOFtiNL;&JM6>)$Td__ zG5A62rEhOOO=0g9@9893I1~wZYXH~rI=8iw3lDH~tv>nEh>xtjDvUxgwhd_%gdei5 zns7nQ?i&>KeAQbyy0wTFQm=bvZkFDl1iOpF;Xxdt#6^pplQYhJU}gN$Kisc;_)NaZbcW} ztC_w$Z>Hx4K#l~^9z(-q*;=IC{U^$TB&M7rC+6-*DUIv+*-!Bg(|Z(*XIuB(`kcLa zdEHAf1VEa6P%a=lz{Arupl&%a2fRf-AZR-lD$*mpDpDMJTA`fSF{M#7o?{*SyR8HA znm4?>25C45B!AXyr(@?4cc6g4LclTd?8FF)f%C)tWV+$s6h{$xt~K$j;bXK?z4=tUgm1#pFk=2|En=RJwV~OE+`{ zP%`^NTI0%bJoQ31gV`S zs6fZzI#2@TF)*cw{J3?+VZVGS-Mr(i8@38s^GGN} zs&~KSkC+B|3V|WxIZZ`gtgDel)T15h$hT4o$@8qHxxFryIPZFkqr0c6IS_A1mguay z>xT4~K+BfB8*#+K!_s0qvcj6$Q7nGKy8f75E|IBLhs?xPf;X(&nw`FjmP!9dTWzjsYvqWt{d3T zVvI6_%}y~aK2cva`Kcr6#3dakq86Um;Pl@xPmpCMFTxzXdB#D!KMAcP0h(qeC%Qu|b5o_AM7tziE zb$zp;9R=iPtW)D-w$ilv$ZJ|rT4i=??Id);J8_>@+?xW$LW~B7so*KYOBJ7#x*?|x z4PjVJ5oV!vex&9$-OKTDap%1pPq(w+KK%4huX{Q8zLNwO?;njBBsgCC)Ojz5z;uz| zP%-~iSQ6Z0BQy8wUXE489rtpy9qfU=LnC7K<()lx4W%KC7OLm0UX{BR$#K5$b{y(>G&-0@F!RMYbdNT=*P1E?cfL3$ zI2T@#$db*9WCc0O6h|x75-T`kPn$vcWxjH9Hw`MvQoJTxXM=}|+)bs+t@X;VJZTu9 zQjDeT$i>SN4CQE{dfwXUhJSn^<$UF(99D6?(?aiILD8h79?H?})+}Y+tPG5>DPw-Nl9K%b z|I5xoq-(;V%$AP$0l`VC|^6e44a|>rEm@%(seVnqk#O1 zb>vG+zmu*0>t!oKAiGF0ib>sclE+IL`%TCszfR@oEibGOh)QQjH#X#YPDlayRqMRN zzN(XQzVlWNcX%KOxeF79)1n^vpAJ;AO<@u{#bN89djN=}_@6HQH^fmye$Dz@b@THQ zXN|Ww%IGLZ>WvCyf@I>Zvs@=CIlawAIQim`v9TY8n5;7^=HeMrL|$f{_`C0x%T{Z> zZABEO15J;V_9pqiyEL4rWR>lvWD9GV$`@NI)$G(SFqNZ-yxh9#{Fm;Qa=!Oc4rl&! zS&ABp3e21b_Y#M@tJ&-3mADXxtq|)N;aK+;Ib25(`E~2+(Yv21aenX?hkPGA+S$`( zq7UsVj^6FU@=9seX)73xaF$8m)LUm)?ua^y$ZuE=44?F_#QD)%95#!vC}@9>zUdwl zFL5YEqQSg!q=?J7FwI610}eBqi+M;P`AzGTruaJ&=O-_5G|fTAwFq1$fCqVrBgU+5 zF3X!(kUqFUOz-hyJx9~##Ntwq;+}6=8-4rq_kV(-yso(aZK5c@(7tt;JK;W$tFqlG z%9Lv1-q`G22S3ncAv-9h9fKXbUnjwfBNn30*-v%w-^8Fyhz)BGIvIlG52B6PV-AX` zeR**9sSf&^u8`)&?+zNall}&0vvbHn4}^W&MHO~j!OC^l!7LG!eVsw6>j7ObnkC$U z$rvrxFM9BkqgifTH~i!-l4vMTJrt}_Pxtp^Ya5*XQKx>$uVY!>ualo#qB?N)xegV< z)5d!`DY{M_8N6SIj%;dSC-2v(A<+&y#~kise9z%Nxpc*D7|Q6b^Pvv+;fq=PG|~-o zwG&f2zv%Ry9D-yw*JMMQ>-eIRp`0sfch_WhQIH+F5d}Gs^&jUo*_sDXEXocn=D(V_^`cJNlbjwffkf)n_vSFWgD7PDe za+bE*9YMM4i`_X;=R+Op$#py@KRMTbeltBetDJ7=$z6AW|7Ln})>=O3$sHaISnI^<3H*#|OsPl$w`0|{6>YkkpH$z7{yxGYeA97(Q!Ih+&Kt)$%53Z1~cZ>+Zia#IWSue|)8 z5l-TAF9Y08y|B{Q{e-oRf7|-`vxzTAoL{`ffqA1QE97EbPUUd}8-`<7q6?7n#9>7D zzvzZ0$aRh)M_AkVcdRp4Ew0ekaAb+ceOi>q;N6j(Q*~kU0|mylScf=Na-kG-BMooU z;a$}1qS=$3xkI{V*L@QBK@9KTJa|_hDn3Cl1bw zu_%yGv(_8kOj~IizruR=wCx_0IHSD9LAxr{Q2_@BT{karC`Re^M%{=*)y8hib9s-d z`ngqwu(t8X+ew^Ak=auAL+}%|kQ|*xL9))%aWNgShHma0Q3DDH6oc&1X!c{)`1h@u+kbJYbTispH{3kR6)n1BmtqPnQg_`%aU49EYLlbHfi%oE6==C# z$$3pRMQz0S!0Pjt=Y~t1F<#ZQ4qk_z$gy znfE;!+kIuGiGX0;UI*0_%XB#+>}>D=6MbyoRuJmLV(a4fpa}4qCNaa*3jf` zi)5;C-lh`El>b$Clbbc^e7Y6KI4TA?1eQ(yBxgi)Gzzoaz{mUEO>q>EKeonRy4B4R z=K?Qr=yR!1LSuk_nXamFugH`B>3xPeQsVR+M(77o!sK0ItTyH(k0>CovaUR*$vzcv zp|`ClABp^`da)Y)?%fl_fiBM5fxD4L=R~0K5JJT=Ji&mImO*1`f<3hV)5tJQ@1n^lwvONRt)o%`Xv#R zrU!x6qh`o;4@^61lzEB~O+y4mj7>$}Wbv^0AUs|tK%$g%{CU}WMK}&2GTH?4TtAzU- z5UIB6s1-fFykLwttYMIV=vRslWCr7ig7H_@k;^ZCK$ObMO5ERg!E!OyL+b0U9F=|v z*t8Cu%F&rV-CPREk!^)z)r=|4OVUivmnJv8vAeWNPuRI}fXBm)#Z-p;iXj~N4fSU3-Quke^ybNoO+(rD z!~I~5-utQfV#Zxp(#ec_$8K+p`{K%kCo?YpdboK15PY|Wmj30`(eyt#%s6o5To1<` zliWPt}p3e#OC)FLxlzl_okJiX1ws}I+`$n(nO*50#h0_f1aJeSl;~gEa zP;eUM&-vm&;pT7;c|JQ2qJ^3K#k5ZSWSuhMsPXdQDf!Cj@?o_;?6*9r+`&Zmpl~A& z>>7H)&ZA-Q>uIC0;b0Yq*yhO5V_K%xSvUTC<2Xrklb1B?rr0HtS!>y&utXZ^YAf~y zWK_hqyctEQ6i{TvXh$sY+^uFz%hb=-De*5x$y8IlOhqQDgVd04aU5jwp#Jj;7xpIF zq2zfEBODJRO`5wjE9Cjgasa)3u_69mwoY&c>xKCw) zypZecSX3R0W=yNpxkYc@yyhTRlKKcVgZJCx@Ymm;TU55oaV_cUOjWx2jPkyNnp)e+ zc_Sz<3@xutH#C$tq+8qe=wJHVU-I|W@f#of7k=>elKS6ee{jmOU(PN1?a%sf=EP6# z;91j3>RYnUTK>#Mql(I^%FF*+*N`diQ&Cry8D2sAq&`#KR#A=jM=A!@HP+QOw}mSQ z)bT3RA&r@eBlyeTjykSiwChi8E$O<(K=X(`e=2X_W96)Oe=4tQEZ=8PQ_YZu%zi$w z4}Ng!F2a6CF80+AT{_AK_WwLN-8Dz;`~}attEB!xD+_$}DEk+Gmy2}3bCD}w4DF3|E;pME#1~yzHfPJreQ#!{ZE12c{F!P<&+ zW4gMju_hg<$TSXZt;meXR5a98RWy%i8`#vy>Envly0%QeW+0|(Gp!YEO)b>}D=OPE zjjc^BIle|hZ!4`+7wth#9}c94v^CYXq-(nL<@8~P{DnpINYC%Y0)TKy z*5%fQY+8Daw;tN<*13DZx}8(f|5mYX+tcCIaJG23#vFxaW_ok~`V_8Q#9kz5{3yiE;uj`g|+xb0*a_sMQVsz+6oYF2B4&^Y|rGRco zw_U-fn>&HK9t+RQ(0RMxbqLhmoxoY`{6^Ak&QCh#midzo{V>?&JWkSWuJioXD}l2( z{{Q0Ka$6SHc5$>x7uGxO1KybXHE7*&?(x^J18!e0e-r06t_b0ua~p~NFFCh-(3^9c zOm@S$Rb9`EbK5??b?K(<_t*}~3~Fi_G0N&e2<03@2Zc5mV3hoYJ3}=uyW`xpk2xu~ zI!lvBxn+IxpxoM{PRcEAl~Quh-G#uM&C^ZDze2ff|BwsimeViHiw?r9lXBaG*NjI0 z7Z(ECpK>MLB4g|?i<1|gOI|s*r0i^uTiO1f7jxhE?%f?8E8_hcB?r#xQQ9*c8!=>X zYgJS4jXMfjL&Ig8bk?@|4Gs4xu5YT^b07ZEZ(v(nb8BT~Wo=#Cz#&zE>ZU=J^+Ou9 zn+!!O^XsDRn`rhu;qr`qA~>9?f!n9|vsDd9R)uS-(#d!_l1Rtm2U|y3CvDh}Zp*Z` zRb5lm)9Ss=uykWhRlim?ooqh)H8(ZXRgdU5G{o)In|1v0@brkLA#J@&`uro@r_a#R zZCWyeht#!X22pu#4Gd^$Iy2K4XdB+vd)wN>st-Q6{-gtj4LT%wT;o9%HD~k{$c8+KRS8&1VjaH7AEP zR`nTF`nz^XQBa`1wW+c9wyB|q9+CRzfro@L@nkyIaO8l44?O-1{oLrDet$00IJl~E zNW8k)I&-8|TiY5RJYZmbOT&nUnn+8qDRgFax;9eNmL4>`Dl)jHWpHTdFicuEaKPZI zA%hbwXGUs=$20WCY(64xg}m5 zsu}{zwY8ykaMgfdZF*QLof(>LP7NPS8>gnVAv4MvX+=XdRiRdj{OPvFP}>MD#u^xI z8a%Y2F*0QM8BOtE-B5bx^;LDPZPDrhncBud$!Pts>R_ZMI&^qjCR|rPaPY9&(2&Ne z`b0zPut?p|+PapSWL2=HHQY8d+8hfGNTh0-V=d9)wQV)g#$?N|!SNvj2DR0t6E(w! H#fJQUX