diff --git a/wandb/debug-cli.chenj0g.log b/wandb/debug-cli.chenj0g.log deleted file mode 100644 index e69de29..0000000 diff --git a/wandb/debug-internal.log b/wandb/debug-internal.log deleted file mode 120000 index 2b1e52e..0000000 --- a/wandb/debug-internal.log +++ /dev/null @@ -1 +0,0 @@ -run-20231023_091030-25yuqsbo/logs/debug-internal.log \ No newline at end of file diff --git a/wandb/debug.log b/wandb/debug.log deleted file mode 120000 index bce2fd2..0000000 --- a/wandb/debug.log +++ /dev/null @@ -1 +0,0 @@ -run-20231023_091030-25yuqsbo/logs/debug.log \ No newline at end of file diff --git a/wandb/latest-run b/wandb/latest-run deleted file mode 120000 index 4482a73..0000000 --- a/wandb/latest-run +++ /dev/null @@ -1 +0,0 @@ -run-20231023_091030-25yuqsbo \ No newline at end of file diff --git a/wandb/run-20231023_085438-bxwpu63q/files/conda-environment.yaml b/wandb/run-20231023_085438-bxwpu63q/files/conda-environment.yaml deleted file mode 100644 index cfced9e..0000000 --- a/wandb/run-20231023_085438-bxwpu63q/files/conda-environment.yaml +++ /dev/null @@ -1,297 +0,0 @@ -name: eye -channels: - - pytorch - - defaults -dependencies: - - _libgcc_mutex=0.1=main - - _openmp_mutex=5.1=1_gnu - - blas=1.0=mkl - - brotlipy=0.7.0=py39h27cfd23_1003 - - bzip2=1.0.8=h7b6447c_0 - - ca-certificates=2023.01.10=h06a4308_0 - - certifi=2022.12.7=py39h06a4308_0 - - cffi=1.15.1=py39h5eee18b_3 - - charset-normalizer=2.0.4=pyhd3eb1b0_0 - - cryptography=39.0.1=py39h9ce1e76_0 - - cudatoolkit=11.3.1=h2bc3f7f_2 - - ffmpeg=4.3=hf484d3e_0 - - flit-core=3.8.0=py39h06a4308_0 - - freetype=2.12.1=h4a9f257_0 - - giflib=5.2.1=h5eee18b_3 - - gmp=6.2.1=h295c915_3 - - gnutls=3.6.15=he1e5248_0 - - idna=3.4=py39h06a4308_0 - - intel-openmp=2021.4.0=h06a4308_3561 - - jpeg=9e=h5eee18b_1 - - lame=3.100=h7b6447c_0 - - lcms2=2.12=h3be6417_0 - - ld_impl_linux-64=2.38=h1181459_1 - - lerc=3.0=h295c915_0 - - libdeflate=1.17=h5eee18b_0 - - libffi=3.4.2=h6a678d5_6 - - libgcc-ng=11.2.0=h1234567_1 - - libgomp=11.2.0=h1234567_1 - - libiconv=1.16=h7f8727e_2 - - libidn2=2.3.2=h7f8727e_0 - - libpng=1.6.39=h5eee18b_0 - - libstdcxx-ng=11.2.0=h1234567_1 - - libtasn1=4.19.0=h5eee18b_0 - - libtiff=4.5.0=h6a678d5_2 - - libunistring=0.9.10=h27cfd23_0 - - libwebp=1.2.4=h11a3e52_1 - - libwebp-base=1.2.4=h5eee18b_1 - - lz4-c=1.9.4=h6a678d5_0 - - mkl=2021.4.0=h06a4308_640 - - mkl-service=2.4.0=py39h7f8727e_0 - - mkl_fft=1.3.1=py39hd3c417c_0 - - mkl_random=1.2.2=py39h51133e4_0 - - ncurses=6.4=h6a678d5_0 - - nettle=3.7.3=hbbd107a_1 - - numpy=1.23.5=py39h14f4228_0 - - numpy-base=1.23.5=py39h31eccc5_0 - - openh264=2.1.1=h4ff587b_0 - - openssl=1.1.1t=h7f8727e_0 - - pillow=9.4.0=py39h6a678d5_0 - - pip=23.0.1=py39h06a4308_0 - - pycparser=2.21=pyhd3eb1b0_0 - - pyopenssl=23.0.0=py39h06a4308_0 - - pysocks=1.7.1=py39h06a4308_0 - - python=3.9.16=h7a1cb2a_2 - - pytorch-mutex=1.0=cuda - - readline=8.2=h5eee18b_0 - - requests=2.28.1=py39h06a4308_1 - - setuptools=65.6.3=py39h06a4308_0 - - six=1.16.0=pyhd3eb1b0_1 - - sqlite=3.41.1=h5eee18b_0 - - tk=8.6.12=h1ccaba5_0 - - torchaudio=0.12.1=py39_cu113 - - torchvision=0.13.1=py39_cu113 - - typing_extensions=4.4.0=py39h06a4308_0 - - urllib3=1.26.15=py39h06a4308_0 - - wheel=0.38.4=py39h06a4308_0 - - xz=5.2.10=h5eee18b_1 - - zlib=1.2.13=h5eee18b_0 - - zstd=1.5.4=hc292b87_0 - - pip: - - accelerate==0.20.3 - - aiofiles==23.1.0 - - aiohttp==3.8.4 - - aiosignal==1.3.1 - - altair==4.2.2 - - antlr4-python3-runtime==4.9.3 - - anyio==3.6.2 - - appdirs==1.4.4 - - argon2-cffi==21.3.0 - - argon2-cffi-bindings==21.2.0 - - arrow==1.2.3 - - asttokens==2.2.1 - - async-timeout==4.0.2 - - attrs==22.2.0 - - backcall==0.2.0 - - beautifulsoup4==4.12.2 - - bitsandbytes==0.37.0 - - bleach==6.0.0 - - blis==0.7.9 - - braceexpand==0.1.7 - - catalogue==2.0.8 - - cchardet==2.1.7 - - chardet==5.1.0 - - click==8.1.3 - - cloudpickle==2.2.1 - - cmake==3.26.1 - - comm==0.1.3 - - confection==0.0.4 - - contourpy==1.0.7 - - cycler==0.11.0 - - cymem==2.0.7 - - datasets==2.13.1 - - debugpy==1.6.7 - - decorator==5.1.1 - - decord==0.6.0 - - defusedxml==0.7.1 - - dill==0.3.6 - - docker-pycreds==0.4.0 - - einops==0.6.0 - - entrypoints==0.4 - - executing==1.2.0 - - fairscale==0.4.13 - - fastapi==0.95.0 - - fastjsonschema==2.16.3 - - ffmpy==0.3.0 - - filelock==3.9.0 - - fire==0.5.0 - - fonttools==4.38.0 - - fqdn==1.5.1 - - frozenlist==1.3.3 - - fsspec==2023.4.0 - - gitdb==4.0.10 - - gitpython==3.1.31 - - gradio==3.24.1 - - gradio-client==0.0.8 - - h11==0.14.0 - - httpcore==0.16.3 - - httpx==0.23.3 - - huggingface-hub==0.16.4 - - imageio==2.31.1 - - importlib-metadata==6.2.0 - - importlib-resources==5.12.0 - - iopath==0.1.10 - - ipykernel==6.22.0 - - ipython==8.12.0 - - ipython-genutils==0.2.0 - - isoduration==20.11.0 - - jedi==0.18.2 - - jinja2==3.1.2 - - joblib==1.2.0 - - jsonpointer==2.3 - - jsonschema==4.17.3 - - jupyter-client==8.1.0 - - jupyter-core==5.3.0 - - jupyter-events==0.6.3 - - jupyter-server==2.5.0 - - jupyter-server-terminals==0.4.4 - - jupyterlab-pygments==0.2.2 - - kiwisolver==1.4.4 - - langcodes==3.3.0 - - lazy-loader==0.2 - - linkify-it-py==2.0.0 - - lit==16.0.0 - - llvmlite==0.39.1 - - loralib==0.1.1 - - markdown-it-py==2.2.0 - - markupsafe==2.1.2 - - matplotlib==3.7.0 - - matplotlib-inline==0.1.6 - - mdit-py-plugins==0.3.3 - - mdurl==0.1.2 - - mistune==2.0.5 - - mpmath==1.3.0 - - multidict==6.0.4 - - multiprocess==0.70.14 - - murmurhash==1.0.9 - - nbclassic==0.5.5 - - nbclient==0.7.3 - - nbconvert==7.3.0 - - nbformat==5.8.0 - - nest-asyncio==1.5.6 - - networkx==3.1 - - nltk==3.8.1 - - notebook==6.5.4 - - notebook-shim==0.2.2 - - numba==0.56.4 - - nvidia-cublas-cu11==11.10.3.66 - - nvidia-cuda-cupti-cu11==11.7.101 - - nvidia-cuda-nvrtc-cu11==11.7.99 - - nvidia-cuda-runtime-cu11==11.7.99 - - nvidia-cudnn-cu11==8.5.0.96 - - nvidia-cufft-cu11==10.9.0.58 - - nvidia-curand-cu11==10.2.10.91 - - nvidia-cusolver-cu11==11.4.0.1 - - nvidia-cusparse-cu11==11.7.4.91 - - nvidia-nccl-cu11==2.14.3 - - nvidia-nvtx-cu11==11.7.91 - - omegaconf==2.3.0 - - openai==0.27.0 - - opencv-python==4.7.0.72 - - orjson==3.8.9 - - packaging==23.0 - - pandas==2.0.0 - - pandocfilters==1.5.0 - - parso==0.8.3 - - pathlib==1.0.1 - - pathtools==0.1.2 - - pathy==0.10.1 - - peft==0.2.0 - - pexpect==4.8.0 - - pickleshare==0.7.5 - - platformdirs==3.2.0 - - policies==0.4.2 - - portalocker==2.7.0 - - preshed==3.0.8 - - progressbar2==4.2.0 - - prometheus-client==0.16.0 - - prompt-toolkit==3.0.38 - - protobuf==4.22.1 - - psutil==5.9.4 - - ptyprocess==0.7.0 - - pure-eval==0.2.2 - - pyarrow==12.0.1 - - pycocoevalcap==1.2 - - pycocotools==2.0.6 - - pycryptodome==3.17 - - pydantic==1.10.7 - - pydub==0.25.1 - - pygments==2.14.0 - - pynndescent==0.5.8 - - pyparsing==3.0.9 - - pyrsistent==0.19.3 - - python-dateutil==2.8.2 - - python-json-logger==2.0.7 - - python-multipart==0.0.6 - - python-utils==3.7.0 - - pytz==2023.3 - - pywavelets==1.4.1 - - pyyaml==6.0 - - pyzmq==25.0.2 - - regex==2022.10.31 - - rfc3339-validator==0.1.4 - - rfc3986==1.5.0 - - rfc3986-validator==0.1.1 - - safetensors==0.3.1 - - scikit-image==0.21.0 - - scikit-learn==1.2.2 - - scipy==1.10.1 - - seaborn==0.12.2 - - semantic-version==2.10.0 - - send2trash==1.8.0 - - sentence-transformers==2.2.2 - - sentencepiece==0.1.97 - - sentry-sdk==1.19.1 - - setproctitle==1.3.2 - - smart-open==6.3.0 - - smmap==5.0.0 - - sniffio==1.3.0 - - soupsieve==2.4 - - spacy==3.5.1 - - spacy-legacy==3.0.12 - - spacy-loggers==1.0.4 - - srsly==2.4.6 - - stack-data==0.6.2 - - starlette==0.26.1 - - submitit==1.4.5 - - sympy==1.11.1 - - tenacity==8.2.2 - - termcolor==2.3.0 - - terminado==0.17.1 - - thinc==8.1.9 - - threadpoolctl==3.1.0 - - tifffile==2023.4.12 - - timm==0.6.13 - - tinycss2==1.2.1 - - tokenizers==0.13.3 - - toolz==0.12.0 - - torch==2.0.0 - - tornado==6.2 - - tqdm==4.64.1 - - traitlets==5.9.0 - - transformers==4.31.0 - - triton==2.0.0 - - typer==0.7.0 - - tzdata==2023.3 - - uc-micro-py==1.0.1 - - umap-learn==0.5.3 - - uri-template==1.2.0 - - uvicorn==0.21.1 - - visual-genome==1.1.1 - - wandb==0.14.2 - - wasabi==1.1.1 - - wcwidth==0.2.6 - - webcolors==1.13 - - webdataset==0.2.48 - - webencodings==0.5.1 - - websocket-client==1.5.1 - - websockets==11.0.1 - - xxhash==3.2.0 - - yarl==1.8.2 - - zipp==3.14.0 -prefix: /home/chenj0g/anaconda3/envs/eye diff --git a/wandb/run-20231023_085438-bxwpu63q/files/config.yaml b/wandb/run-20231023_085438-bxwpu63q/files/config.yaml deleted file mode 100644 index 8f9f09d..0000000 --- a/wandb/run-20231023_085438-bxwpu63q/files/config.yaml +++ /dev/null @@ -1,38 +0,0 @@ -wandb_version: 1 - -_wandb: - desc: null - value: - python_version: 3.9.16 - cli_version: 0.14.2 - framework: huggingface - huggingface_version: 4.31.0 - is_jupyter_run: false - is_kaggle_kernel: false - start_time: 1698040478.151963 - t: - 1: - - 1 - - 11 - - 41 - - 49 - - 55 - - 63 - - 71 - 2: - - 1 - - 11 - - 41 - - 49 - - 55 - - 63 - - 71 - 3: - - 1 - - 13 - - 23 - 4: 3.9.16 - 5: 0.14.2 - 6: 4.31.0 - 8: - - 5 diff --git a/wandb/run-20231023_085438-bxwpu63q/files/output.log b/wandb/run-20231023_085438-bxwpu63q/files/output.log deleted file mode 100644 index 92a24d8..0000000 --- a/wandb/run-20231023_085438-bxwpu63q/files/output.log +++ /dev/null @@ -1,177 +0,0 @@ - -2023-10-23 08:54:52,781 [INFO] Start training -batch sizes [[2]] -module.llama_model.base_model.model.model.layers.0.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.0.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.0.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.0.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.1.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.1.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.1.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.1.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.2.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.2.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.2.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.2.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.3.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.3.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.3.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.3.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.4.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.4.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.4.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.4.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.5.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.5.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.5.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.5.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.6.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.6.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.6.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.6.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.7.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.7.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.7.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.7.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.8.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.8.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.8.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.8.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.9.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.9.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.9.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.9.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.10.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.10.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.10.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.10.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.11.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.11.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.11.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.11.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.12.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.12.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.12.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.12.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.13.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.13.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.13.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.13.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.14.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.14.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.14.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.14.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.15.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.15.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.15.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.15.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.16.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.16.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.16.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.16.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.17.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.17.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.17.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.17.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.18.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.18.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.18.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.18.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.19.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.19.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.19.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.19.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.20.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.20.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.20.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.20.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.21.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.21.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.21.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.21.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.22.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.22.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.22.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.22.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.23.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.23.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.23.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.23.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.24.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.24.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.24.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.24.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.25.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.25.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.25.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.25.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.26.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.26.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.26.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.26.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.27.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.27.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.27.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.27.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.28.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.28.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.28.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.28.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.29.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.29.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.29.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.29.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.30.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.30.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.30.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.30.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.31.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.31.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.31.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.31.self_attn.v_proj.lora_B.weight -module.llama_proj.weight -module.llama_proj.bias -2023-10-23 08:54:54,750 [INFO] dataset_ratios not specified, datasets will be concatenated (map-style datasets) or chained (webdataset.DataPipeline). -2023-10-23 08:54:54,750 [INFO] Loaded 12171 records for train split from the dataset. -2023-10-23 08:54:54,768 [INFO] number of trainable parameters: 56627200 -2023-10-23 08:54:54,769 [INFO] Start training epoch 0, 1000 iters per inner epoch. -Train: data epoch: [0] [ 0/1000] eta: 0:44:54 lr: 0.000001 loss: 1.3049 time: 2.6941 data: 0.0000 max mem: 33053 -/home/chenj0g/anaconda3/envs/eye/lib/python3.9/site-packages/torch/utils/checkpoint.py:31: UserWarning: None of the inputs have requires_grad=True. Gradients will be None - warnings.warn("None of the inputs have requires_grad=True. Gradients will be None") -Train: data epoch: [0] [ 50/1000] eta: 0:07:57 lr: 0.000001 loss: 1.2327 time: 0.4510 data: 0.0000 max mem: 38294 -Train: data epoch: [0] [ 100/1000] eta: 0:07:10 lr: 0.000002 loss: 1.1049 time: 0.4456 data: 0.0000 max mem: 39573 -Train: data epoch: [0] [ 150/1000] eta: 0:06:42 lr: 0.000002 loss: 1.2879 time: 0.4630 data: 0.0000 max mem: 40208 -Train: data epoch: [0] [ 200/1000] eta: 0:06:18 lr: 0.000003 loss: 1.0077 time: 0.4521 data: 0.0000 max mem: 40208 -Train: data epoch: [0] [ 250/1000] eta: 0:05:52 lr: 0.000003 loss: 1.4790 time: 0.4627 data: 0.0000 max mem: 41447 -Train: data epoch: [0] [ 300/1000] eta: 0:05:27 lr: 0.000004 loss: 1.2864 time: 0.4646 data: 0.0000 max mem: 41447 -Train: data epoch: [0] [ 350/1000] eta: 0:05:02 lr: 0.000004 loss: 1.1261 time: 0.4556 data: 0.0000 max mem: 41447 -Traceback (most recent call last): - File "/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/train.py", line 117, in - main() - File "/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/train.py", line 113, in main - runner.train() - File "/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/minigpt4/runners/runner_base.py", line 377, in train - train_stats = self.train_epoch(cur_epoch) - File "/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/minigpt4/runners/runner_base.py", line 437, in train_epoch - return self.task.train_epoch( - File "/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/minigpt4/tasks/base_task.py", line 116, in train_epoch - return self._train_inner_loop( - File "/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/minigpt4/tasks/base_task.py", line 221, in _train_inner_loop - loss = self.train_step(model=model, samples=samples) - File "/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/minigpt4/tasks/base_task.py", line 70, in train_step - loss = model(samples)["loss"] - File "/home/chenj0g/anaconda3/envs/eye/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl - return forward_call(*args, **kwargs) - File "/home/chenj0g/anaconda3/envs/eye/lib/python3.9/site-packages/torch/nn/parallel/distributed.py", line 1156, in forward - output = self._run_ddp_forward(*inputs, **kwargs) - File "/home/chenj0g/anaconda3/envs/eye/lib/python3.9/site-packages/torch/nn/parallel/distributed.py", line 1110, in _run_ddp_forward - return module_to_run(*inputs[0], **kwargs[0]) # type: ignore[index] - File "/home/chenj0g/anaconda3/envs/eye/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl - return forward_call(*args, **kwargs) - File "/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/minigpt4/models/minigpt_base.py", line 279, in forward - self.preparing_embedding(samples) - File "/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/minigpt4/models/minigpt_base.py", line 215, in preparing_embedding - img_embeds, img_atts = self.encode_img(samples["image"]) - File "/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/minigpt4/models/minigpt_v2.py", line 88, in encode_img - atts_llama = torch.ones(inputs_llama.size()[:-1], dtype=torch.long).to(image.device) -KeyboardInterrupt \ No newline at end of file diff --git a/wandb/run-20231023_085438-bxwpu63q/files/requirements.txt b/wandb/run-20231023_085438-bxwpu63q/files/requirements.txt deleted file mode 100644 index 75bc393..0000000 --- a/wandb/run-20231023_085438-bxwpu63q/files/requirements.txt +++ /dev/null @@ -1,246 +0,0 @@ -accelerate==0.20.3 -aiofiles==23.2.1 -aiohttp==3.8.4 -aiosignal==1.3.1 -altair==5.1.2 -antlr4-python3-runtime==4.9.3 -anyio==3.6.2 -appdirs==1.4.4 -argon2-cffi-bindings==21.2.0 -argon2-cffi==21.3.0 -arrow==1.2.3 -asttokens==2.2.1 -async-timeout==4.0.2 -attrs==22.2.0 -backcall==0.2.0 -beautifulsoup4==4.12.2 -bitsandbytes==0.37.0 -bleach==6.0.0 -blis==0.7.9 -braceexpand==0.1.7 -brotlipy==0.7.0 -catalogue==2.0.8 -cchardet==2.1.7 -certifi==2022.12.7 -cffi==1.15.1 -chardet==5.1.0 -charset-normalizer==2.0.4 -click==8.1.3 -cloudpickle==2.2.1 -cmake==3.26.1 -comm==0.1.3 -confection==0.0.4 -contourpy==1.0.7 -cryptography==39.0.1 -cycler==0.11.0 -cymem==2.0.7 -datasets==2.13.1 -debugpy==1.6.7 -decorator==5.1.1 -decord==0.6.0 -defusedxml==0.7.1 -dill==0.3.6 -docker-pycreds==0.4.0 -einops==0.6.0 -entrypoints==0.4 -executing==1.2.0 -fairscale==0.4.13 -fastapi==0.95.0 -fastjsonschema==2.16.3 -ffmpy==0.3.1 -filelock==3.9.0 -fire==0.5.0 -flit-core==3.8.0 -fonttools==4.38.0 -fqdn==1.5.1 -frozenlist==1.3.3 -fsspec==2023.9.2 -gitdb==4.0.10 -gitpython==3.1.31 -gradio-client==0.6.0 -gradio==3.24.1 -h11==0.14.0 -httpcore==0.18.0 -httpx==0.25.0 -huggingface-hub==0.18.0 -idna==3.4 -imageio==2.31.1 -importlib-metadata==6.2.0 -importlib-resources==5.12.0 -iopath==0.1.10 -ipykernel==6.22.0 -ipython-genutils==0.2.0 -ipython==8.12.0 -isoduration==20.11.0 -jedi==0.18.2 -jinja2==3.1.2 -joblib==1.2.0 -jsonpointer==2.3 -jsonschema==4.17.3 -jupyter-client==8.1.0 -jupyter-core==5.3.0 -jupyter-events==0.6.3 -jupyter-server-terminals==0.4.4 -jupyter-server==2.5.0 -jupyterlab-pygments==0.2.2 -kiwisolver==1.4.4 -langcodes==3.3.0 -lazy-loader==0.2 -linkify-it-py==2.0.2 -lit==16.0.0 -llvmlite==0.39.1 -loralib==0.1.1 -markdown-it-py==2.2.0 -markupsafe==2.1.2 -matplotlib-inline==0.1.6 -matplotlib==3.7.0 -mdit-py-plugins==0.3.3 -mdurl==0.1.2 -mistune==2.0.5 -mkl-fft==1.3.1 -mkl-random==1.2.2 -mkl-service==2.4.0 -mpmath==1.3.0 -multidict==6.0.4 -multiprocess==0.70.14 -murmurhash==1.0.9 -nbclassic==0.5.5 -nbclient==0.7.3 -nbconvert==7.3.0 -nbformat==5.8.0 -nest-asyncio==1.5.6 -networkx==3.1 -nltk==3.8.1 -notebook-shim==0.2.2 -notebook==6.5.4 -numba==0.56.4 -numpy==1.23.5 -nvidia-cublas-cu11==11.10.3.66 -nvidia-cuda-cupti-cu11==11.7.101 -nvidia-cuda-nvrtc-cu11==11.7.99 -nvidia-cuda-runtime-cu11==11.7.99 -nvidia-cudnn-cu11==8.5.0.96 -nvidia-cufft-cu11==10.9.0.58 -nvidia-curand-cu11==10.2.10.91 -nvidia-cusolver-cu11==11.4.0.1 -nvidia-cusparse-cu11==11.7.4.91 -nvidia-nccl-cu11==2.14.3 -nvidia-nvtx-cu11==11.7.91 -omegaconf==2.3.0 -openai==0.27.0 -opencv-python==4.7.0.72 -orjson==3.9.8 -packaging==23.0 -pandas==2.0.0 -pandocfilters==1.5.0 -parso==0.8.3 -pathlib==1.0.1 -pathtools==0.1.2 -pathy==0.10.1 -peft==0.2.0 -pexpect==4.8.0 -pickleshare==0.7.5 -pillow==9.4.0 -pip==23.0.1 -platformdirs==3.2.0 -policies==0.4.2 -portalocker==2.7.0 -preshed==3.0.8 -progressbar2==4.2.0 -prometheus-client==0.16.0 -prompt-toolkit==3.0.38 -protobuf==4.22.1 -psutil==5.9.4 -ptyprocess==0.7.0 -pure-eval==0.2.2 -pyarrow==12.0.1 -pycocoevalcap==1.2 -pycocotools==2.0.6 -pycparser==2.21 -pycryptodome==3.17 -pydantic==1.10.7 -pydub==0.25.1 -pygments==2.14.0 -pynndescent==0.5.8 -pyopenssl==23.0.0 -pyparsing==3.0.9 -pyrsistent==0.19.3 -pysocks==1.7.1 -python-dateutil==2.8.2 -python-json-logger==2.0.7 -python-multipart==0.0.6 -python-utils==3.7.0 -pytz==2023.3 -pywavelets==1.4.1 -pyyaml==6.0 -pyzmq==25.0.2 -regex==2022.10.31 -requests==2.28.1 -rfc3339-validator==0.1.4 -rfc3986-validator==0.1.1 -rfc3986==1.5.0 -rpe-index==1.2.0 -safetensors==0.3.1 -scikit-image==0.21.0 -scikit-learn==1.2.2 -scipy==1.10.1 -seaborn==0.12.2 -semantic-version==2.10.0 -send2trash==1.8.0 -sentence-transformers==2.2.2 -sentencepiece==0.1.97 -sentry-sdk==1.19.1 -setproctitle==1.3.2 -setuptools==65.6.3 -six==1.16.0 -smart-open==6.3.0 -smmap==5.0.0 -sniffio==1.3.0 -soupsieve==2.4 -spacy-legacy==3.0.12 -spacy-loggers==1.0.4 -spacy==3.5.1 -srsly==2.4.6 -stack-data==0.6.2 -starlette==0.26.1 -submitit==1.4.5 -sympy==1.11.1 -tenacity==8.2.2 -termcolor==2.3.0 -terminado==0.17.1 -thinc==8.1.9 -threadpoolctl==3.1.0 -tifffile==2023.4.12 -timm==0.6.13 -tinycss2==1.2.1 -tokenizers==0.13.3 -toolz==0.12.0 -torch==2.0.0 -torchaudio==0.12.1 -torchvision==0.13.1 -tornado==6.2 -tqdm==4.64.1 -traitlets==5.9.0 -transformers==4.31.0 -triton==2.0.0 -typer==0.7.0 -typing-extensions==4.4.0 -tzdata==2023.3 -uc-micro-py==1.0.2 -umap-learn==0.5.3 -uri-template==1.2.0 -urllib3==1.26.15 -uvicorn==0.21.1 -visual-genome==1.1.1 -wandb==0.14.2 -wasabi==1.1.1 -wcwidth==0.2.6 -webcolors==1.13 -webdataset==0.2.48 -webencodings==0.5.1 -websocket-client==1.5.1 -websockets==11.0.1 -wheel==0.38.4 -xxhash==3.2.0 -yarl==1.8.2 -zipp==3.14.0 \ No newline at end of file diff --git a/wandb/run-20231023_085438-bxwpu63q/files/wandb-metadata.json b/wandb/run-20231023_085438-bxwpu63q/files/wandb-metadata.json deleted file mode 100644 index 0b7590d..0000000 --- a/wandb/run-20231023_085438-bxwpu63q/files/wandb-metadata.json +++ /dev/null @@ -1,374 +0,0 @@ -{ - "os": "Linux-3.10.0-1160.76.1.el7.x86_64-x86_64-with-glibc2.17", - "python": "3.9.16", - "heartbeatAt": "2023-10-23T05:54:39.620251", - "startedAt": "2023-10-23T05:54:38.092408", - "docker": null, - "cuda": null, - "args": [ - "--job_name", - "minigpt4_v2_test", - "--cfg-path", - "train_configs/minigpt_v2_finetune.yaml" - ], - "state": "running", - "program": "/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/train.py", - "codePath": "train.py", - "git": { - "remote": "https://github.com/junchen14/MiniGPT-4_finetune", - "commit": "5c3ec8bb730b2e9df23b4bb3eb24108f9c04a8d6" - }, - "email": "jun.chen@kaust.edu.sa", - "root": "/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune", - "host": "gpu201-16-l", - "username": "chenj0g", - "executable": "/home/chenj0g/anaconda3/envs/eye/bin/python", - "cpu_count": 64, - "cpu_count_logical": 64, - "cpu_freq": { - "current": 1996.2770000000016, - "min": 0.0, - "max": 0.0 - }, - "cpu_freq_per_core": [ - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - } - ], - "disk": { - "total": 251.85773849487305, - "used": 7.9049072265625 - }, - "gpu": "NVIDIA A100-SXM4-80GB", - "gpu_count": 2, - "gpu_devices": [ - { - "name": "NVIDIA A100-SXM4-80GB", - "memory_total": 85899345920 - }, - { - "name": "NVIDIA A100-SXM4-80GB", - "memory_total": 85899345920 - } - ], - "memory": { - "total": 503.7154769897461 - } -} diff --git a/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json b/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json deleted file mode 100644 index 8c4dc99..0000000 --- a/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json +++ /dev/null @@ -1 +0,0 @@ -{"epoch": 0, "loss": 1.1899254322052002, "_timestamp": 1698040660.278502, "_runtime": 182.1265389919281, "_step": 355, "_wandb": {"runtime": 180}} \ No newline at end of file diff --git a/wandb/run-20231023_085438-bxwpu63q/logs/debug-internal.log b/wandb/run-20231023_085438-bxwpu63q/logs/debug-internal.log deleted file mode 100644 index ac6f9de..0000000 --- a/wandb/run-20231023_085438-bxwpu63q/logs/debug-internal.log +++ /dev/null @@ -1,1843 +0,0 @@ -2023-10-23 08:54:38,141 INFO StreamThr :49933 [internal.py:wandb_internal():86] W&B internal server running at pid: 49933, started at: 2023-10-23 08:54:38.140580 -2023-10-23 08:54:38,142 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status -2023-10-23 08:54:38,153 INFO WriterThread:49933 [datastore.py:open_for_write():85] open: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/run-bxwpu63q.wandb -2023-10-23 08:54:38,154 DEBUG SenderThread:49933 [sender.py:send():375] send: header -2023-10-23 08:54:38,561 DEBUG SenderThread:49933 [sender.py:send():375] send: run -2023-10-23 08:54:39,255 INFO SenderThread:49933 [dir_watcher.py:__init__():219] watching files in: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files -2023-10-23 08:54:39,255 INFO SenderThread:49933 [sender.py:_start_run_threads():1124] run started: bxwpu63q with start time 1698040478.151963 -2023-10-23 08:54:39,255 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:54:39,256 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:54:39,257 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: check_version -2023-10-23 08:54:39,257 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: check_version -2023-10-23 08:54:39,599 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: run_start -2023-10-23 08:54:39,604 DEBUG HandlerThread:49933 [system_info.py:__init__():31] System info init -2023-10-23 08:54:39,604 DEBUG HandlerThread:49933 [system_info.py:__init__():46] System info init done -2023-10-23 08:54:39,604 INFO HandlerThread:49933 [system_monitor.py:start():181] Starting system monitor -2023-10-23 08:54:39,604 INFO SystemMonitor:49933 [system_monitor.py:_start():145] Starting system asset monitoring threads -2023-10-23 08:54:39,604 INFO HandlerThread:49933 [system_monitor.py:probe():201] Collecting system info -2023-10-23 08:54:39,605 INFO SystemMonitor:49933 [interfaces.py:start():190] Started cpu monitoring -2023-10-23 08:54:39,605 INFO SystemMonitor:49933 [interfaces.py:start():190] Started disk monitoring -2023-10-23 08:54:39,605 INFO SystemMonitor:49933 [interfaces.py:start():190] Started gpu monitoring -2023-10-23 08:54:39,606 INFO SystemMonitor:49933 [interfaces.py:start():190] Started memory monitoring -2023-10-23 08:54:39,606 INFO SystemMonitor:49933 [interfaces.py:start():190] Started network monitoring -2023-10-23 08:54:39,620 DEBUG HandlerThread:49933 [system_info.py:probe():195] Probing system -2023-10-23 08:54:39,624 DEBUG HandlerThread:49933 [system_info.py:_probe_git():180] Probing git -2023-10-23 08:54:39,635 DEBUG HandlerThread:49933 [system_info.py:_probe_git():188] Probing git done -2023-10-23 08:54:39,635 DEBUG HandlerThread:49933 [system_info.py:probe():240] Probing system done -2023-10-23 08:54:39,635 DEBUG HandlerThread:49933 [system_monitor.py:probe():210] {'os': 'Linux-3.10.0-1160.76.1.el7.x86_64-x86_64-with-glibc2.17', 'python': '3.9.16', 'heartbeatAt': '2023-10-23T05:54:39.620251', 'startedAt': '2023-10-23T05:54:38.092408', 'docker': None, 'cuda': None, 'args': ('--job_name', 'minigpt4_v2_test', '--cfg-path', 'train_configs/minigpt_v2_finetune.yaml'), 'state': 'running', 'program': '/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/train.py', 'codePath': 'train.py', 'git': {'remote': 'https://github.com/junchen14/MiniGPT-4_finetune', 'commit': '5c3ec8bb730b2e9df23b4bb3eb24108f9c04a8d6'}, 'email': 'jun.chen@kaust.edu.sa', 'root': '/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune', 'host': 'gpu201-16-l', 'username': 'chenj0g', 'executable': '/home/chenj0g/anaconda3/envs/eye/bin/python', 'cpu_count': 64, 'cpu_count_logical': 64, 'cpu_freq': {'current': 1996.2770000000016, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}], 'disk': {'total': 251.85773849487305, 'used': 7.9049072265625}, 'gpu': 'NVIDIA A100-SXM4-80GB', 'gpu_count': 2, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}], 'memory': {'total': 503.7154769897461}} -2023-10-23 08:54:39,635 INFO HandlerThread:49933 [system_monitor.py:probe():211] Finished collecting system info -2023-10-23 08:54:39,635 INFO HandlerThread:49933 [system_monitor.py:probe():214] Publishing system info -2023-10-23 08:54:39,635 DEBUG HandlerThread:49933 [system_info.py:_save_pip():51] Saving list of pip packages installed into the current environment -2023-10-23 08:54:39,636 DEBUG HandlerThread:49933 [system_info.py:_save_pip():67] Saving pip packages done -2023-10-23 08:54:39,636 DEBUG HandlerThread:49933 [system_info.py:_save_conda():74] Saving list of conda packages installed into the current environment -2023-10-23 08:54:40,257 INFO Thread-12 :49933 [dir_watcher.py:_on_file_created():278] file/dir created: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/requirements.txt -2023-10-23 08:54:40,257 INFO Thread-12 :49933 [dir_watcher.py:_on_file_created():278] file/dir created: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/conda-environment.yaml -2023-10-23 08:54:40,257 INFO Thread-12 :49933 [dir_watcher.py:_on_file_created():278] file/dir created: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:54:52,732 DEBUG HandlerThread:49933 [system_info.py:_save_conda():86] Saving conda packages done -2023-10-23 08:54:52,734 INFO HandlerThread:49933 [system_monitor.py:probe():216] Finished publishing system info -2023-10-23 08:54:52,767 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:54:52,767 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: keepalive -2023-10-23 08:54:52,767 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:54:52,767 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: keepalive -2023-10-23 08:54:52,768 DEBUG SenderThread:49933 [sender.py:send():375] send: files -2023-10-23 08:54:52,768 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-metadata.json with policy now -2023-10-23 08:54:52,773 DEBUG SenderThread:49933 [sender.py:send():375] send: telemetry -2023-10-23 08:54:52,773 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 08:54:52,774 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: stop_status -2023-10-23 08:54:53,158 DEBUG SenderThread:49933 [sender.py:send():375] send: telemetry -2023-10-23 08:54:53,264 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/conda-environment.yaml -2023-10-23 08:54:53,265 INFO Thread-12 :49933 [dir_watcher.py:_on_file_created():278] file/dir created: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/output.log -2023-10-23 08:54:53,265 INFO Thread-12 :49933 [dir_watcher.py:_on_file_created():278] file/dir created: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-metadata.json -2023-10-23 08:54:53,539 INFO wandb-upload_0:49933 [upload_job.py:push():137] Uploaded file /tmp/tmpgw7o7efowandb/ck5f4n94-wandb-metadata.json -2023-10-23 08:54:54,159 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:54:55,284 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/output.log -2023-10-23 08:54:57,285 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/output.log -2023-10-23 08:54:57,463 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:54:57,464 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:54:57,465 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:54:57,466 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:54:57,900 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:54:57,901 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:54:57,901 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:54:57,902 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:54:58,286 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:54:58,411 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:54:58,412 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:54:58,412 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:54:58,413 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:54:58,806 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:54:58,806 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:54:58,806 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:54:58,807 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:54:59,237 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:54:59,237 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:54:59,238 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:54:59,238 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:54:59,238 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:54:59,287 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/output.log -2023-10-23 08:54:59,287 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:54:59,820 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:54:59,821 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:54:59,821 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:54:59,822 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:00,263 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:00,263 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:00,264 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:00,264 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:00,287 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:00,721 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:00,722 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:00,722 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:00,723 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:01,220 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:01,220 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:01,220 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:01,222 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:01,288 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:01,648 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:01,648 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:01,648 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:01,649 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:02,094 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:02,094 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:02,095 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:02,096 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:02,289 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:02,557 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:02,558 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:02,558 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:02,559 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:02,939 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:02,939 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:02,939 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:02,940 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:03,290 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:03,443 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:03,444 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:03,444 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:03,446 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:03,865 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:03,865 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:03,866 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:03,866 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:04,290 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:04,408 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:04,409 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:04,409 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:04,409 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:55:04,410 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:04,777 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:04,777 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:04,777 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:04,778 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:05,291 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:05,329 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:05,329 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:05,329 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:05,330 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:05,833 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:05,833 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:05,833 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:05,835 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:06,289 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:06,289 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:06,289 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:06,290 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:06,292 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:06,715 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:06,716 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:06,716 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:06,717 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:07,166 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:07,167 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:07,167 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:07,168 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:07,292 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:07,711 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:07,711 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:07,711 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:07,712 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:07,773 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 08:55:07,773 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: stop_status -2023-10-23 08:55:08,171 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:08,172 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:08,172 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:08,174 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:08,293 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:08,614 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:08,614 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:08,615 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:08,615 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:09,077 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:09,078 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:09,078 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:09,079 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:09,294 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:09,504 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:09,505 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:09,508 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:55:09,746 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:09,746 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:09,957 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:09,958 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:09,958 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:09,958 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:10,295 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/config.yaml -2023-10-23 08:55:10,295 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:10,419 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:10,420 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:10,420 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:10,422 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:10,856 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:10,857 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:10,857 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:10,858 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:11,295 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:11,409 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:11,410 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:11,410 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:11,411 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:11,858 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:11,858 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:11,858 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:11,859 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:12,296 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:12,399 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:12,400 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:12,400 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:12,401 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:12,782 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:12,783 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:12,783 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:12,785 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:13,206 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:13,207 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:13,207 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:13,212 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:13,297 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:13,591 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:13,592 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:13,592 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:13,593 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:14,044 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:14,045 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:14,045 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:14,046 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:14,297 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:14,498 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:14,498 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:14,499 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:14,499 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:14,976 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:14,976 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:14,977 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:14,978 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:55:14,979 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:15,298 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:15,451 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:15,451 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:15,451 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:15,452 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:15,888 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:15,888 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:15,888 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:15,889 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:16,273 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:16,274 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:16,274 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:16,275 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:16,299 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:16,756 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:16,757 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:16,757 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:16,758 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:17,201 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:17,201 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:17,201 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:17,203 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:17,299 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:17,636 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:17,636 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:17,637 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:17,638 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:18,145 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:18,146 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:18,146 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:18,147 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:18,300 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:18,546 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:18,547 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:18,547 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:18,548 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:18,985 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:18,986 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:18,986 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:18,987 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:19,305 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:19,490 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:19,491 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:19,491 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:19,493 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:19,933 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:19,933 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:19,934 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:19,934 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:20,305 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:20,428 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:20,429 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:20,429 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:20,430 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:55:20,430 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:20,875 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:20,876 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:20,876 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:20,877 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:21,302 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:21,302 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:21,302 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:21,303 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:21,306 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:21,736 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:21,737 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:21,737 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:21,739 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:22,180 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:22,181 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:22,181 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:22,182 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:22,307 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:22,668 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:22,668 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:22,668 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:22,669 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:22,776 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 08:55:22,776 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: stop_status -2023-10-23 08:55:23,106 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:23,107 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:23,107 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:23,108 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:23,307 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/output.log -2023-10-23 08:55:23,308 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:23,534 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:23,534 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:23,535 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:23,535 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:24,001 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:24,002 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:24,002 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:24,004 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:24,308 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:24,366 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:24,366 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:24,367 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:24,367 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:24,827 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:24,827 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:24,828 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:24,828 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:25,309 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:25,322 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:25,323 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:25,323 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:25,324 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:25,766 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:25,766 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:25,767 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:25,767 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:55:25,768 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:26,310 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:26,319 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:26,320 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:26,320 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:26,322 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:26,795 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:26,795 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:26,796 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:26,796 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:27,243 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:27,243 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:27,244 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:27,244 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:27,310 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:27,698 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:27,699 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:27,699 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:27,700 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:28,121 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:28,122 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:28,122 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:28,123 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:28,311 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:28,564 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:28,565 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:28,565 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:28,566 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:29,131 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:29,131 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:29,131 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:29,132 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:29,312 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:29,575 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:29,576 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:29,576 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:29,577 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:30,038 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:30,038 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:30,038 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:30,039 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:30,312 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:30,557 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:30,557 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:30,558 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:30,558 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:30,980 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:30,981 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:30,981 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:30,982 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:55:30,983 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:31,313 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:31,485 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:31,486 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:31,486 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:31,487 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:31,960 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:31,960 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:31,960 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:31,961 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:32,314 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:32,430 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:32,431 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:32,431 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:32,432 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:32,860 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:32,861 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:32,861 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:32,862 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:33,314 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:33,320 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:33,320 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:33,320 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:33,322 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:33,706 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:33,706 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:33,706 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:33,707 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:34,180 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:34,181 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:34,181 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:34,182 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:34,315 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:34,649 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:34,650 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:34,650 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:34,651 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:35,152 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:35,152 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:35,152 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:35,153 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:35,316 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:35,574 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:35,575 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:35,575 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:35,577 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:35,944 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:35,945 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:35,945 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:35,946 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:36,317 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:36,373 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:36,373 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:36,374 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:36,374 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:55:36,375 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:36,842 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:36,842 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:36,842 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:36,843 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:37,289 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:37,290 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:37,290 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:37,291 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:37,317 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:37,797 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 08:55:37,798 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: stop_status -2023-10-23 08:55:37,801 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:38,094 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:38,095 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:38,096 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:38,167 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:38,168 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:38,168 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:38,169 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:38,318 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:38,562 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:38,563 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:38,563 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:38,564 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:39,040 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:39,041 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:39,041 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:39,042 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:39,318 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:39,479 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:39,480 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:39,480 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:39,481 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:39,606 DEBUG SystemMonitor:49933 [system_monitor.py:_start():159] Starting system metrics aggregation loop -2023-10-23 08:55:39,608 DEBUG SenderThread:49933 [sender.py:send():375] send: stats -2023-10-23 08:55:39,951 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:39,951 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:39,952 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:39,952 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:40,319 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:40,383 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:40,384 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:40,384 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:40,385 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:40,806 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:40,807 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:40,807 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:40,808 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:41,203 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:41,204 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:41,204 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:41,205 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:41,320 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:41,639 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:41,640 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:41,640 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:41,640 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:55:41,641 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:42,253 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:42,254 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:42,254 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:42,255 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:42,321 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:42,693 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:42,694 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:42,694 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:42,695 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:43,092 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:43,093 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:43,093 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:43,094 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:43,321 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:43,636 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:43,637 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:43,637 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:43,638 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:44,130 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:44,130 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:44,130 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:44,131 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:44,322 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:44,514 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:44,515 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:44,515 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:44,516 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:44,972 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:44,972 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:44,972 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:44,973 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:45,323 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/output.log -2023-10-23 08:55:45,323 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:45,448 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:45,449 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:45,449 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:45,450 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:46,031 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:46,032 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:46,032 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:46,033 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:46,323 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:46,601 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:46,602 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:46,602 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:46,603 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:47,062 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:47,062 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:47,063 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:47,063 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:55:47,064 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:47,324 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:47,476 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:47,477 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:47,477 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:47,478 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:47,941 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:47,942 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:47,942 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:47,943 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:48,325 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:48,418 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:48,419 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:48,419 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:48,421 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:48,832 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:48,833 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:48,833 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:48,834 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:49,298 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:49,298 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:49,298 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:49,299 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:49,325 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:49,693 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:49,693 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:49,694 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:49,694 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:50,169 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:50,169 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:50,170 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:50,170 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:50,326 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:50,615 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:50,616 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:50,616 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:50,617 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:51,077 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:51,077 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:51,078 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:51,078 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:51,327 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:51,523 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:51,523 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:51,524 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:51,524 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:51,964 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:51,964 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:51,964 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:51,965 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:52,328 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:52,448 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:52,448 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:52,448 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:52,448 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:55:52,449 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:52,824 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 08:55:52,825 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: stop_status -2023-10-23 08:55:52,828 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:53,090 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:53,091 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:53,091 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:53,303 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:53,303 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:53,303 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:53,304 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:53,329 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:53,780 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:53,781 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:53,781 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:53,782 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:54,253 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:54,253 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:54,253 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:54,254 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:54,329 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:54,715 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:54,716 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:54,716 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:54,716 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:55,267 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:55,268 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:55,268 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:55,269 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:55,330 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:55,712 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:55,712 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:55,713 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:55,713 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:56,123 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:56,124 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:56,124 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:56,125 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:56,331 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:56,598 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:56,599 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:56,599 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:56,600 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:57,017 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:57,017 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:57,018 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:57,020 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:57,331 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:57,522 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:57,523 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:57,523 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:57,523 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:55:57,524 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:57,966 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:57,967 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:57,967 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:57,968 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:58,332 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:58,394 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:58,395 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:58,395 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:58,396 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:58,841 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:58,842 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:58,842 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:58,844 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:59,237 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:59,237 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:59,237 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:59,238 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:55:59,333 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:55:59,661 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:55:59,662 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:55:59,662 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:55:59,663 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:00,169 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:00,170 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:00,170 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:00,171 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:00,333 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:00,598 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:00,598 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:00,598 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:00,599 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:01,037 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:01,037 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:01,038 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:01,038 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:01,334 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:01,505 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:01,506 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:01,506 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:01,507 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:01,964 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:01,965 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:01,965 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:01,966 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:02,335 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:02,458 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:02,458 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:02,459 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:02,459 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:03,003 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:03,004 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:03,004 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:03,004 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:56:03,005 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:03,335 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:03,509 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:03,509 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:03,509 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:03,510 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:04,013 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:04,014 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:04,014 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:04,015 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:04,336 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:04,501 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:04,501 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:04,502 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:04,502 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:05,007 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:05,008 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:05,008 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:05,009 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:05,337 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:05,368 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:05,369 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:05,369 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:05,370 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:05,842 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:05,842 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:05,843 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:05,843 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:06,278 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:06,278 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:06,279 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:06,279 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:06,338 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:06,726 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:06,726 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:06,726 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:06,727 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:07,244 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:07,244 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:07,244 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:07,245 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:07,338 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/output.log -2023-10-23 08:56:07,338 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:07,801 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:07,802 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:07,802 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:07,803 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:07,824 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 08:56:07,824 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: stop_status -2023-10-23 08:56:08,067 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:56:08,238 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:08,239 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:08,239 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:08,240 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:08,339 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:08,634 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:08,634 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:08,634 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:08,635 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:09,178 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:09,179 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:09,179 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:09,181 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:09,340 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:09,609 DEBUG SenderThread:49933 [sender.py:send():375] send: stats -2023-10-23 08:56:09,736 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:09,736 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:09,737 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:09,737 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:10,341 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:10,347 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:10,348 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:10,348 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:10,349 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:10,802 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:10,802 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:10,802 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:10,803 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:11,304 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:11,304 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:11,304 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:11,305 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:11,341 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:11,848 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:11,849 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:11,849 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:11,850 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:12,233 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:12,234 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:12,234 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:12,235 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:12,342 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:12,700 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:12,701 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:12,701 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:12,702 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:13,160 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:13,161 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:13,161 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:13,161 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:56:13,162 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:13,343 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:13,679 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:13,679 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:13,680 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:13,680 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:14,231 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:14,232 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:14,232 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:14,233 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:14,343 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:14,633 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:14,633 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:14,633 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:14,634 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:15,076 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:15,077 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:15,077 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:15,078 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:15,344 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:15,537 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:15,537 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:15,537 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:15,538 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:16,057 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:16,057 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:16,057 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:16,058 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:16,345 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:16,553 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:16,553 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:16,553 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:16,554 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:16,938 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:16,939 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:16,939 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:16,939 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:17,346 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:17,396 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:17,397 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:17,397 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:17,398 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:17,875 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:17,875 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:17,875 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:17,876 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:18,346 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:18,351 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:18,351 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:18,352 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:18,352 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:56:18,352 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:18,834 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:18,834 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:18,834 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:18,835 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:19,296 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:19,297 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:19,297 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:19,299 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:19,347 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:19,760 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:19,761 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:19,761 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:19,762 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:20,210 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:20,210 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:20,211 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:20,211 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:20,348 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:20,711 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:20,711 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:20,711 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:20,712 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:21,219 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:21,219 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:21,219 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:21,220 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:21,348 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:21,666 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:21,666 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:21,666 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:21,667 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:22,136 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:22,136 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:22,136 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:22,137 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:22,349 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:22,575 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:22,575 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:22,575 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:22,576 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:22,824 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 08:56:22,824 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: stop_status -2023-10-23 08:56:22,970 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:23,033 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:23,033 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:23,034 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:23,350 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:23,400 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:23,400 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:23,401 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:23,401 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:56:23,401 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:23,858 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:23,859 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:23,859 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:23,859 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:24,235 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:24,235 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:24,235 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:24,236 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:24,350 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:24,606 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:24,606 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:24,606 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:24,608 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:25,070 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:25,070 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:25,071 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:25,071 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:25,351 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:25,408 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:25,409 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:25,409 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:25,409 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:25,982 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:25,983 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:25,983 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:25,984 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:26,352 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:26,477 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:26,477 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:26,477 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:26,478 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:26,855 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:26,856 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:26,856 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:26,857 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:27,320 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:27,320 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:27,320 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:27,321 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:27,353 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:27,813 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:27,814 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:27,814 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:27,815 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:28,250 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:28,251 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:28,251 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:28,252 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:28,353 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:28,793 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:28,794 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:28,794 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:28,794 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:56:28,795 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:29,239 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:29,239 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:29,239 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:29,240 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:29,354 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:29,753 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:29,753 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:29,754 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:29,756 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:30,214 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:30,214 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:30,214 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:30,215 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:30,355 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:30,645 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:30,646 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:30,646 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:30,647 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:31,079 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:31,079 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:31,079 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:31,080 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:31,355 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/output.log -2023-10-23 08:56:31,356 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:31,559 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:31,559 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:31,560 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:31,560 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:32,005 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:32,006 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:32,006 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:32,007 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:32,356 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:32,416 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:32,417 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:32,417 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:32,418 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:32,887 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:32,887 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:32,887 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:32,888 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:33,357 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:33,364 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:33,364 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:33,365 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:33,365 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:33,877 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:33,877 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:33,878 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:33,878 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:56:33,878 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:34,322 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:34,323 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:34,323 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:34,324 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:34,357 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:34,791 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:34,791 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:34,791 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:34,792 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:35,253 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:35,253 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:35,253 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:35,254 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:35,358 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:35,829 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:35,829 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:35,829 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:35,830 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:36,288 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:36,288 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:36,288 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:36,289 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:36,359 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:36,651 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:36,651 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:36,652 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:36,652 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:37,109 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:37,110 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:37,110 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:37,110 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:37,360 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:37,552 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:37,553 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:37,553 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:37,554 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:37,824 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 08:56:37,824 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: stop_status -2023-10-23 08:56:37,938 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:38,058 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:38,058 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:38,059 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:38,360 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:38,384 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:38,385 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:38,385 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:38,386 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:38,766 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:38,766 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:38,767 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:38,767 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:39,310 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:39,311 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:39,311 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:39,311 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:56:39,312 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:39,361 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:39,610 DEBUG SenderThread:49933 [sender.py:send():375] send: stats -2023-10-23 08:56:39,694 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:39,694 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:39,694 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:39,695 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:40,315 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:40,316 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:40,316 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:40,318 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:40,362 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:40,700 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:40,700 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:40,700 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:40,701 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:41,072 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:41,073 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:41,073 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:41,074 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:41,363 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:41,441 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:41,441 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:41,441 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:41,442 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:41,997 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:41,998 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:41,998 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:41,999 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:42,363 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:42,518 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:42,518 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:42,519 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:42,519 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:42,945 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:42,945 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:42,945 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:42,946 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:43,364 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:43,370 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:43,371 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:43,371 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:43,372 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:43,949 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:43,950 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:43,950 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:43,951 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:44,365 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:44,417 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:44,417 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:44,418 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:44,418 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:56:44,418 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:44,871 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:44,872 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:44,872 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:44,873 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:45,266 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:45,266 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:45,266 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:45,267 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:45,365 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:45,713 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:45,714 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:45,714 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:45,715 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:46,189 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:46,190 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:46,190 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:46,191 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:46,366 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:46,813 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:46,814 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:46,814 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:46,815 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:47,246 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:47,246 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:47,246 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:47,247 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:47,367 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:47,691 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:47,692 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:47,692 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:47,692 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:48,168 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:48,168 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:48,169 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:48,169 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:48,368 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:48,636 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:48,636 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:48,636 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:48,637 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:49,007 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:49,008 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:49,008 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:49,009 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:49,368 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:49,559 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:49,560 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:49,560 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:49,560 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:56:49,561 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:49,999 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:50,000 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:50,000 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:50,001 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:50,369 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:50,444 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:50,445 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:50,445 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:50,447 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:50,891 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:50,892 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:50,892 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:50,892 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:51,356 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:51,356 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:51,357 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:51,357 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:51,370 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:51,800 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:51,800 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:51,800 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:51,801 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:52,222 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:52,223 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:52,223 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:52,224 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:52,370 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:52,625 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:52,626 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:52,626 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:52,626 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:52,824 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 08:56:52,824 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: stop_status -2023-10-23 08:56:53,127 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:53,128 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:53,128 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:53,129 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:53,371 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:53,564 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:53,564 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:53,565 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:53,565 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:54,010 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:54,010 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:54,010 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:54,011 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:54,372 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:54,462 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:54,462 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:54,462 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:54,463 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:54,922 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:54,922 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:54,922 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:54,923 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:56:54,923 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:55,372 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/output.log -2023-10-23 08:56:55,373 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:55,397 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:55,398 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:55,398 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:55,399 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:55,793 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:55,793 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:55,793 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:55,794 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:56,221 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:56,222 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:56,222 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:56,223 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:56,373 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:56,724 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:56,724 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:56,724 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:56,725 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:57,198 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:57,199 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:57,199 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:57,200 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:57,374 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:57,582 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:57,583 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:57,583 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:57,584 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:58,104 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:58,105 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:58,105 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:58,106 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:58,375 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:58,583 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:58,583 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:58,584 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:58,584 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:59,050 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:59,050 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:59,050 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:59,051 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:59,375 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:56:59,506 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:59,506 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:59,506 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:59,507 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:56:59,899 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:56:59,900 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:56:59,900 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:56:59,901 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:00,339 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:00,339 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:00,339 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:00,339 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:57:00,340 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:00,376 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:00,815 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:00,815 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:00,815 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:00,817 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:01,244 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:01,244 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:01,244 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:01,245 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:01,377 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:01,685 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:01,685 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:01,685 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:01,686 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:02,058 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:02,058 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:02,058 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:02,059 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:02,378 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:02,532 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:02,533 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:02,533 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:02,534 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:03,067 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:03,068 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:03,068 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:03,069 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:03,378 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:03,445 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:03,446 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:03,446 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:03,446 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:03,865 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:03,865 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:03,866 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:03,866 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:04,264 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:04,265 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:04,265 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:04,266 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:04,379 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:04,770 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:04,771 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:04,771 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:04,772 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:05,295 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:05,296 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:05,296 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:05,297 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:05,380 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:05,740 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:05,740 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:05,740 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:05,741 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:57:05,741 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:06,189 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:06,189 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:06,189 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:06,190 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:06,381 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:06,625 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:06,625 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:06,625 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:06,626 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:07,015 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:07,015 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:07,015 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:07,016 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:07,381 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:07,443 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:07,443 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:07,444 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:07,444 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:07,824 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 08:57:07,824 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: stop_status -2023-10-23 08:57:07,988 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:08,025 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:08,025 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:08,026 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:08,382 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:08,503 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:08,503 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:08,503 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:08,504 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:09,023 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:09,024 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:09,024 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:09,025 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:09,383 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:09,581 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:09,581 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:09,581 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:09,582 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:09,611 DEBUG SenderThread:49933 [sender.py:send():375] send: stats -2023-10-23 08:57:10,096 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:10,097 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:10,097 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:10,098 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:10,383 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:10,544 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:10,545 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:10,545 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:10,545 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:10,991 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:10,992 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:10,992 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:10,993 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:57:10,994 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:11,384 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:11,446 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:11,447 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:11,447 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:11,448 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:11,822 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:11,822 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:11,822 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:11,823 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:12,296 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:12,296 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:12,297 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:12,298 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:12,385 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:12,699 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:12,700 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:12,700 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:12,701 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:13,126 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:13,126 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:13,126 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:13,127 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:13,386 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:13,717 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:13,717 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:13,717 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:13,718 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:14,196 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:14,197 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:14,197 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:14,197 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:14,386 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:14,642 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:14,642 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:14,642 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:14,643 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:15,075 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:15,076 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:15,076 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:15,077 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:15,387 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:15,480 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:15,480 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:15,481 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:15,481 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:15,958 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:15,959 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:15,959 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:15,960 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:16,387 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:16,387 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:16,388 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:16,388 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:57:16,388 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:16,389 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:16,864 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:16,864 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:16,864 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:16,865 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:17,305 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:17,306 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:17,306 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:17,306 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:17,389 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/output.log -2023-10-23 08:57:17,389 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:17,693 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:17,693 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:17,693 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:17,694 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:18,116 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:18,117 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:18,117 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:18,120 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:18,390 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:18,539 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:18,540 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:18,540 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:18,541 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:18,934 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:18,934 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:18,934 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:18,935 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:19,323 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:19,323 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:19,323 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:19,324 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:19,390 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:19,750 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:19,751 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:19,751 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:19,752 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:20,332 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:20,333 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:20,333 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:20,334 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:20,391 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:20,809 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:20,810 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:20,810 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:20,811 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:21,260 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:21,261 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:21,261 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:21,263 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:21,392 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:21,696 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:21,697 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:21,697 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:21,697 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:57:21,698 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:22,144 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:22,144 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:22,144 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:22,145 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:22,393 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:22,603 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:22,603 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:22,604 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:22,604 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:22,824 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 08:57:22,824 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: stop_status -2023-10-23 08:57:23,048 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:23,049 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:23,049 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:23,049 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:23,393 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:23,490 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:23,491 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:23,491 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:23,492 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:23,913 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:23,913 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:23,913 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:23,914 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:24,394 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:24,442 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:24,442 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:24,442 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:24,443 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:24,880 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:24,881 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:24,881 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:24,882 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:25,325 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:25,325 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:25,325 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:25,326 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:25,395 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:25,722 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:25,722 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:25,722 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:25,723 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:26,151 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:26,152 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:26,152 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:26,153 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:26,395 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:26,674 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:26,675 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:26,675 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:26,676 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:27,104 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:27,105 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:27,105 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:27,105 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:57:27,106 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:27,396 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:27,631 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:27,632 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:27,632 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:27,633 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:28,074 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:28,075 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:28,075 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:28,076 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:28,397 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:28,497 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:28,498 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:28,498 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:28,499 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:28,949 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:28,949 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:28,950 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:28,950 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:29,397 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:29,419 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:29,419 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:29,420 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:29,420 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:29,980 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:29,980 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:29,981 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:29,981 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:30,380 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:30,380 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:30,381 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:30,381 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:30,398 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:30,968 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:30,969 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:30,969 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:30,970 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:31,344 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:31,345 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:31,345 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:31,347 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:31,399 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:31,774 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:31,775 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:31,775 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:31,776 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:32,222 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:32,222 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:32,223 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:32,223 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:57:32,224 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:32,399 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:32,663 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:32,663 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:32,664 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:32,664 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:33,145 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:33,146 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:33,146 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:33,147 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:33,400 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:33,590 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:33,591 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:33,591 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:33,591 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:34,072 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:34,072 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:34,073 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:34,073 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:34,401 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:34,556 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:34,556 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:34,556 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:34,557 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:34,972 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:34,973 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:34,973 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:34,974 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:35,401 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:35,418 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:35,419 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:35,419 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:35,420 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:35,866 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:35,867 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:35,867 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:35,868 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:36,335 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:36,335 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:36,335 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:36,336 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:36,402 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:36,753 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:36,753 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:36,754 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:36,754 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:37,219 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:37,219 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:37,219 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:37,220 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:37,403 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:37,683 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:37,684 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:37,684 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:37,684 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:57:37,685 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:37,824 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 08:57:37,824 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: stop_status -2023-10-23 08:57:38,062 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:38,062 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:38,062 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:38,063 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:38,403 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:38,485 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:38,485 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:38,486 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:38,486 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:38,929 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:38,930 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:38,930 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:38,931 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:39,375 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:39,375 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:39,376 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:39,377 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:39,404 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/output.log -2023-10-23 08:57:39,404 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:39,612 DEBUG SenderThread:49933 [sender.py:send():375] send: stats -2023-10-23 08:57:39,816 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:39,816 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:39,816 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:39,817 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:40,280 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 08:57:40,281 DEBUG SenderThread:49933 [sender.py:send():375] send: history -2023-10-23 08:57:40,281 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: summary_record -2023-10-23 08:57:40,282 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:40,359 DEBUG SenderThread:49933 [sender.py:send():375] send: exit -2023-10-23 08:57:40,359 INFO SenderThread:49933 [sender.py:send_exit():598] handling exit code: 255 -2023-10-23 08:57:40,360 INFO SenderThread:49933 [sender.py:send_exit():600] handling runtime: 180 -2023-10-23 08:57:40,360 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:40,360 INFO SenderThread:49933 [sender.py:send_exit():606] send defer -2023-10-23 08:57:40,361 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: defer -2023-10-23 08:57:40,361 INFO HandlerThread:49933 [handler.py:handle_request_defer():170] handle defer: 0 -2023-10-23 08:57:40,361 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: defer -2023-10-23 08:57:40,361 INFO SenderThread:49933 [sender.py:send_request_defer():622] handle sender defer: 0 -2023-10-23 08:57:40,361 INFO SenderThread:49933 [sender.py:transition_state():626] send defer: 1 -2023-10-23 08:57:40,361 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: defer -2023-10-23 08:57:40,361 INFO HandlerThread:49933 [handler.py:handle_request_defer():170] handle defer: 1 -2023-10-23 08:57:40,361 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: defer -2023-10-23 08:57:40,361 INFO SenderThread:49933 [sender.py:send_request_defer():622] handle sender defer: 1 -2023-10-23 08:57:40,361 INFO SenderThread:49933 [sender.py:transition_state():626] send defer: 2 -2023-10-23 08:57:40,361 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: defer -2023-10-23 08:57:40,361 INFO HandlerThread:49933 [handler.py:handle_request_defer():170] handle defer: 2 -2023-10-23 08:57:40,361 INFO HandlerThread:49933 [system_monitor.py:finish():190] Stopping system monitor -2023-10-23 08:57:40,362 DEBUG SystemMonitor:49933 [system_monitor.py:_start():166] Finished system metrics aggregation loop -2023-10-23 08:57:40,362 INFO HandlerThread:49933 [interfaces.py:finish():202] Joined cpu monitor -2023-10-23 08:57:40,362 DEBUG SystemMonitor:49933 [system_monitor.py:_start():170] Publishing last batch of metrics -2023-10-23 08:57:40,362 INFO HandlerThread:49933 [interfaces.py:finish():202] Joined disk monitor -2023-10-23 08:57:40,408 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:40,474 INFO HandlerThread:49933 [interfaces.py:finish():202] Joined gpu monitor -2023-10-23 08:57:40,474 INFO HandlerThread:49933 [interfaces.py:finish():202] Joined memory monitor -2023-10-23 08:57:40,474 INFO HandlerThread:49933 [interfaces.py:finish():202] Joined network monitor -2023-10-23 08:57:40,475 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: defer -2023-10-23 08:57:40,475 INFO SenderThread:49933 [sender.py:send_request_defer():622] handle sender defer: 2 -2023-10-23 08:57:40,475 INFO SenderThread:49933 [sender.py:transition_state():626] send defer: 3 -2023-10-23 08:57:40,475 DEBUG SenderThread:49933 [sender.py:send():375] send: stats -2023-10-23 08:57:40,475 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: defer -2023-10-23 08:57:40,475 INFO HandlerThread:49933 [handler.py:handle_request_defer():170] handle defer: 3 -2023-10-23 08:57:40,475 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: defer -2023-10-23 08:57:40,475 INFO SenderThread:49933 [sender.py:send_request_defer():622] handle sender defer: 3 -2023-10-23 08:57:40,476 INFO SenderThread:49933 [sender.py:transition_state():626] send defer: 4 -2023-10-23 08:57:40,476 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: defer -2023-10-23 08:57:40,476 INFO HandlerThread:49933 [handler.py:handle_request_defer():170] handle defer: 4 -2023-10-23 08:57:40,476 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: defer -2023-10-23 08:57:40,476 INFO SenderThread:49933 [sender.py:send_request_defer():622] handle sender defer: 4 -2023-10-23 08:57:40,476 INFO SenderThread:49933 [sender.py:transition_state():626] send defer: 5 -2023-10-23 08:57:40,476 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: defer -2023-10-23 08:57:40,476 INFO HandlerThread:49933 [handler.py:handle_request_defer():170] handle defer: 5 -2023-10-23 08:57:40,476 DEBUG SenderThread:49933 [sender.py:send():375] send: summary -2023-10-23 08:57:40,477 INFO SenderThread:49933 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 08:57:40,477 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: defer -2023-10-23 08:57:40,477 INFO SenderThread:49933 [sender.py:send_request_defer():622] handle sender defer: 5 -2023-10-23 08:57:40,477 INFO SenderThread:49933 [sender.py:transition_state():626] send defer: 6 -2023-10-23 08:57:40,477 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: defer -2023-10-23 08:57:40,477 INFO HandlerThread:49933 [handler.py:handle_request_defer():170] handle defer: 6 -2023-10-23 08:57:40,477 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: defer -2023-10-23 08:57:40,477 INFO SenderThread:49933 [sender.py:send_request_defer():622] handle sender defer: 6 -2023-10-23 08:57:40,477 INFO SenderThread:49933 [sender.py:transition_state():626] send defer: 7 -2023-10-23 08:57:40,478 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:57:40,478 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: defer -2023-10-23 08:57:40,478 INFO HandlerThread:49933 [handler.py:handle_request_defer():170] handle defer: 7 -2023-10-23 08:57:40,478 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: defer -2023-10-23 08:57:40,478 INFO SenderThread:49933 [sender.py:send_request_defer():622] handle sender defer: 7 -2023-10-23 08:57:41,360 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: poll_exit -2023-10-23 08:57:41,409 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/output.log -2023-10-23 08:57:41,409 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:42,956 INFO SenderThread:49933 [sender.py:transition_state():626] send defer: 8 -2023-10-23 08:57:42,956 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: poll_exit -2023-10-23 08:57:42,956 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: defer -2023-10-23 08:57:42,956 INFO HandlerThread:49933 [handler.py:handle_request_defer():170] handle defer: 8 -2023-10-23 08:57:42,956 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: defer -2023-10-23 08:57:42,956 INFO SenderThread:49933 [sender.py:send_request_defer():622] handle sender defer: 8 -2023-10-23 08:57:43,015 INFO SenderThread:49933 [sender.py:transition_state():626] send defer: 9 -2023-10-23 08:57:43,015 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: defer -2023-10-23 08:57:43,015 INFO HandlerThread:49933 [handler.py:handle_request_defer():170] handle defer: 9 -2023-10-23 08:57:43,015 DEBUG SenderThread:49933 [sender.py:send():375] send: artifact -2023-10-23 08:57:43,410 INFO Thread-12 :49933 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/output.log -2023-10-23 08:57:44,837 INFO wandb-upload_1:49933 [upload_job.py:push():95] Uploaded file /home/chenj0g/.local/share/wandb/artifacts/staging/tmp3h8gxbk8 -2023-10-23 08:57:44,842 INFO wandb-upload_0:49933 [upload_job.py:push():95] Uploaded file /home/chenj0g/.local/share/wandb/artifacts/staging/tmpv6vdw94w -2023-10-23 08:57:46,226 INFO SenderThread:49933 [sender.py:send_artifact():1474] sent artifact job-https___github.com_junchen14_MiniGPT-4_finetune_train.py - {'id': 'QXJ0aWZhY3Q6NjEzMjczODU5', 'digest': 'e1357229d7190882bf8b2dba86a37a56', 'state': 'PENDING', 'aliases': [], 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjEwOTExOTI4MQ==', 'latestArtifact': None}, 'version': 'latest'} -2023-10-23 08:57:46,226 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: defer -2023-10-23 08:57:46,226 INFO SenderThread:49933 [sender.py:send_request_defer():622] handle sender defer: 9 -2023-10-23 08:57:46,226 INFO SenderThread:49933 [dir_watcher.py:finish():365] shutting down directory watcher -2023-10-23 08:57:46,360 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: keepalive -2023-10-23 08:57:46,411 INFO SenderThread:49933 [dir_watcher.py:finish():395] scan: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files -2023-10-23 08:57:46,411 INFO SenderThread:49933 [dir_watcher.py:finish():409] scan save: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/config.yaml config.yaml -2023-10-23 08:57:46,411 INFO SenderThread:49933 [dir_watcher.py:finish():409] scan save: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/requirements.txt requirements.txt -2023-10-23 08:57:46,411 INFO SenderThread:49933 [dir_watcher.py:finish():409] scan save: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-metadata.json wandb-metadata.json -2023-10-23 08:57:46,414 INFO SenderThread:49933 [dir_watcher.py:finish():409] scan save: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/conda-environment.yaml conda-environment.yaml -2023-10-23 08:57:46,414 INFO SenderThread:49933 [dir_watcher.py:finish():409] scan save: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/output.log output.log -2023-10-23 08:57:46,416 INFO SenderThread:49933 [dir_watcher.py:finish():409] scan save: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json wandb-summary.json -2023-10-23 08:57:46,416 INFO SenderThread:49933 [sender.py:transition_state():626] send defer: 10 -2023-10-23 08:57:46,416 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: defer -2023-10-23 08:57:46,417 INFO HandlerThread:49933 [handler.py:handle_request_defer():170] handle defer: 10 -2023-10-23 08:57:46,421 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: defer -2023-10-23 08:57:46,423 INFO SenderThread:49933 [sender.py:send_request_defer():622] handle sender defer: 10 -2023-10-23 08:57:46,423 INFO SenderThread:49933 [file_pusher.py:finish():168] shutting down file pusher -2023-10-23 08:57:46,933 INFO wandb-upload_0:49933 [upload_job.py:push():137] Uploaded file /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/requirements.txt -2023-10-23 08:57:47,293 INFO wandb-upload_1:49933 [upload_job.py:push():137] Uploaded file /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/config.yaml -2023-10-23 08:57:47,381 INFO wandb-upload_4:49933 [upload_job.py:push():137] Uploaded file /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/wandb-summary.json -2023-10-23 08:57:47,442 INFO wandb-upload_2:49933 [upload_job.py:push():137] Uploaded file /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/conda-environment.yaml -2023-10-23 08:57:47,443 INFO wandb-upload_3:49933 [upload_job.py:push():137] Uploaded file /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/files/output.log -2023-10-23 08:57:47,643 INFO Thread-11 :49933 [sender.py:transition_state():626] send defer: 11 -2023-10-23 08:57:47,643 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: defer -2023-10-23 08:57:47,644 INFO HandlerThread:49933 [handler.py:handle_request_defer():170] handle defer: 11 -2023-10-23 08:57:47,644 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: defer -2023-10-23 08:57:47,644 INFO SenderThread:49933 [sender.py:send_request_defer():622] handle sender defer: 11 -2023-10-23 08:57:47,644 INFO SenderThread:49933 [file_pusher.py:join():173] waiting for file pusher -2023-10-23 08:57:47,644 INFO SenderThread:49933 [sender.py:transition_state():626] send defer: 12 -2023-10-23 08:57:47,644 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: defer -2023-10-23 08:57:47,644 INFO HandlerThread:49933 [handler.py:handle_request_defer():170] handle defer: 12 -2023-10-23 08:57:47,644 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: defer -2023-10-23 08:57:47,644 INFO SenderThread:49933 [sender.py:send_request_defer():622] handle sender defer: 12 -2023-10-23 08:57:47,845 INFO SenderThread:49933 [sender.py:transition_state():626] send defer: 13 -2023-10-23 08:57:47,845 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: defer -2023-10-23 08:57:47,845 INFO HandlerThread:49933 [handler.py:handle_request_defer():170] handle defer: 13 -2023-10-23 08:57:47,846 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: defer -2023-10-23 08:57:47,846 INFO SenderThread:49933 [sender.py:send_request_defer():622] handle sender defer: 13 -2023-10-23 08:57:47,846 INFO SenderThread:49933 [sender.py:transition_state():626] send defer: 14 -2023-10-23 08:57:47,846 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: defer -2023-10-23 08:57:47,846 INFO HandlerThread:49933 [handler.py:handle_request_defer():170] handle defer: 14 -2023-10-23 08:57:47,846 DEBUG SenderThread:49933 [sender.py:send():375] send: final -2023-10-23 08:57:47,846 DEBUG SenderThread:49933 [sender.py:send():375] send: footer -2023-10-23 08:57:47,846 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: defer -2023-10-23 08:57:47,846 INFO SenderThread:49933 [sender.py:send_request_defer():622] handle sender defer: 14 -2023-10-23 08:57:47,847 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: poll_exit -2023-10-23 08:57:47,847 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: poll_exit -2023-10-23 08:57:47,847 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: server_info -2023-10-23 08:57:47,847 DEBUG SenderThread:49933 [sender.py:send_request():402] send_request: server_info -2023-10-23 08:57:47,849 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: get_summary -2023-10-23 08:57:47,849 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: sampled_history -2023-10-23 08:57:48,027 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 08:57:48,027 INFO MainThread:49933 [wandb_run.py:_footer_history_summary_info():3476] rendering history -2023-10-23 08:57:48,027 INFO MainThread:49933 [wandb_run.py:_footer_history_summary_info():3508] rendering summary -2023-10-23 08:57:48,028 INFO MainThread:49933 [wandb_run.py:_footer_sync_info():3434] logging synced files -2023-10-23 08:57:48,028 DEBUG HandlerThread:49933 [handler.py:handle_request():144] handle_request: shutdown -2023-10-23 08:57:48,028 INFO HandlerThread:49933 [handler.py:finish():845] shutting down handler -2023-10-23 08:57:49,027 INFO SenderThread:49933 [sender.py:finish():1550] shutting down sender -2023-10-23 08:57:49,027 INFO SenderThread:49933 [file_pusher.py:finish():168] shutting down file pusher -2023-10-23 08:57:49,027 INFO SenderThread:49933 [file_pusher.py:join():173] waiting for file pusher -2023-10-23 08:57:49,027 INFO WriterThread:49933 [datastore.py:close():298] close: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/run-bxwpu63q.wandb diff --git a/wandb/run-20231023_085438-bxwpu63q/logs/debug.log b/wandb/run-20231023_085438-bxwpu63q/logs/debug.log deleted file mode 100644 index d243e71..0000000 --- a/wandb/run-20231023_085438-bxwpu63q/logs/debug.log +++ /dev/null @@ -1,27 +0,0 @@ -2023-10-23 08:54:38,127 INFO MainThread:49300 [wandb_setup.py:_flush():76] Configure stats pid to 49300 -2023-10-23 08:54:38,127 INFO MainThread:49300 [wandb_setup.py:_flush():76] Loading settings from /home/chenj0g/.config/wandb/settings -2023-10-23 08:54:38,127 INFO MainThread:49300 [wandb_setup.py:_flush():76] Loading settings from /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/settings -2023-10-23 08:54:38,127 INFO MainThread:49300 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} -2023-10-23 08:54:38,127 INFO MainThread:49300 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'train.py', 'program': '/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/train.py'} -2023-10-23 08:54:38,127 INFO MainThread:49300 [wandb_init.py:_log_setup():507] Logging user logs to /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/logs/debug.log -2023-10-23 08:54:38,127 INFO MainThread:49300 [wandb_init.py:_log_setup():508] Logging internal logs to /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_085438-bxwpu63q/logs/debug-internal.log -2023-10-23 08:54:38,127 INFO MainThread:49300 [wandb_init.py:init():547] calling init triggers -2023-10-23 08:54:38,127 INFO MainThread:49300 [wandb_init.py:init():554] wandb.init called with sweep_config: {} -config: {} -2023-10-23 08:54:38,127 INFO MainThread:49300 [wandb_init.py:init():595] starting backend -2023-10-23 08:54:38,127 INFO MainThread:49300 [wandb_init.py:init():599] setting up manager -2023-10-23 08:54:38,129 INFO MainThread:49300 [backend.py:_multiprocessing_setup():106] multiprocessing start_methods=fork,spawn,forkserver, using: spawn -2023-10-23 08:54:38,151 INFO MainThread:49300 [wandb_init.py:init():605] backend started and connected -2023-10-23 08:54:38,153 INFO MainThread:49300 [wandb_init.py:init():695] updated telemetry -2023-10-23 08:54:38,561 INFO MainThread:49300 [wandb_init.py:init():732] communicating run to backend with 60.0 second timeout -2023-10-23 08:54:39,256 INFO MainThread:49300 [wandb_run.py:_on_init():2176] communicating current version -2023-10-23 08:54:39,594 INFO MainThread:49300 [wandb_run.py:_on_init():2185] got version response upgrade_message: "wandb version 0.15.12 is available! To upgrade, please run:\n $ pip install wandb --upgrade" - -2023-10-23 08:54:39,594 INFO MainThread:49300 [wandb_init.py:init():782] starting run threads in backend -2023-10-23 08:54:52,772 INFO MainThread:49300 [wandb_run.py:_console_start():2157] atexit reg -2023-10-23 08:54:52,772 INFO MainThread:49300 [wandb_run.py:_redirect():2012] redirect: SettingsConsole.WRAP_RAW -2023-10-23 08:54:52,772 INFO MainThread:49300 [wandb_run.py:_redirect():2077] Wrapping output streams. -2023-10-23 08:54:52,772 INFO MainThread:49300 [wandb_run.py:_redirect():2102] Redirects installed. -2023-10-23 08:54:52,773 INFO MainThread:49300 [wandb_init.py:init():824] run started, returning control to user process -2023-10-23 08:54:52,773 INFO MainThread:49300 [wandb_watch.py:watch():51] Watching -2023-10-23 08:57:49,249 WARNING MsgRouterThr:49300 [router.py:message_loop():77] message_loop has been closed diff --git a/wandb/run-20231023_085438-bxwpu63q/run-bxwpu63q.wandb b/wandb/run-20231023_085438-bxwpu63q/run-bxwpu63q.wandb deleted file mode 100644 index 6517852..0000000 Binary files a/wandb/run-20231023_085438-bxwpu63q/run-bxwpu63q.wandb and /dev/null differ diff --git a/wandb/run-20231023_090058-141dzr9c/files/conda-environment.yaml b/wandb/run-20231023_090058-141dzr9c/files/conda-environment.yaml deleted file mode 100644 index cfced9e..0000000 --- a/wandb/run-20231023_090058-141dzr9c/files/conda-environment.yaml +++ /dev/null @@ -1,297 +0,0 @@ -name: eye -channels: - - pytorch - - defaults -dependencies: - - _libgcc_mutex=0.1=main - - _openmp_mutex=5.1=1_gnu - - blas=1.0=mkl - - brotlipy=0.7.0=py39h27cfd23_1003 - - bzip2=1.0.8=h7b6447c_0 - - ca-certificates=2023.01.10=h06a4308_0 - - certifi=2022.12.7=py39h06a4308_0 - - cffi=1.15.1=py39h5eee18b_3 - - charset-normalizer=2.0.4=pyhd3eb1b0_0 - - cryptography=39.0.1=py39h9ce1e76_0 - - cudatoolkit=11.3.1=h2bc3f7f_2 - - ffmpeg=4.3=hf484d3e_0 - - flit-core=3.8.0=py39h06a4308_0 - - freetype=2.12.1=h4a9f257_0 - - giflib=5.2.1=h5eee18b_3 - - gmp=6.2.1=h295c915_3 - - gnutls=3.6.15=he1e5248_0 - - idna=3.4=py39h06a4308_0 - - intel-openmp=2021.4.0=h06a4308_3561 - - jpeg=9e=h5eee18b_1 - - lame=3.100=h7b6447c_0 - - lcms2=2.12=h3be6417_0 - - ld_impl_linux-64=2.38=h1181459_1 - - lerc=3.0=h295c915_0 - - libdeflate=1.17=h5eee18b_0 - - libffi=3.4.2=h6a678d5_6 - - libgcc-ng=11.2.0=h1234567_1 - - libgomp=11.2.0=h1234567_1 - - libiconv=1.16=h7f8727e_2 - - libidn2=2.3.2=h7f8727e_0 - - libpng=1.6.39=h5eee18b_0 - - libstdcxx-ng=11.2.0=h1234567_1 - - libtasn1=4.19.0=h5eee18b_0 - - libtiff=4.5.0=h6a678d5_2 - - libunistring=0.9.10=h27cfd23_0 - - libwebp=1.2.4=h11a3e52_1 - - libwebp-base=1.2.4=h5eee18b_1 - - lz4-c=1.9.4=h6a678d5_0 - - mkl=2021.4.0=h06a4308_640 - - mkl-service=2.4.0=py39h7f8727e_0 - - mkl_fft=1.3.1=py39hd3c417c_0 - - mkl_random=1.2.2=py39h51133e4_0 - - ncurses=6.4=h6a678d5_0 - - nettle=3.7.3=hbbd107a_1 - - numpy=1.23.5=py39h14f4228_0 - - numpy-base=1.23.5=py39h31eccc5_0 - - openh264=2.1.1=h4ff587b_0 - - openssl=1.1.1t=h7f8727e_0 - - pillow=9.4.0=py39h6a678d5_0 - - pip=23.0.1=py39h06a4308_0 - - pycparser=2.21=pyhd3eb1b0_0 - - pyopenssl=23.0.0=py39h06a4308_0 - - pysocks=1.7.1=py39h06a4308_0 - - python=3.9.16=h7a1cb2a_2 - - pytorch-mutex=1.0=cuda - - readline=8.2=h5eee18b_0 - - requests=2.28.1=py39h06a4308_1 - - setuptools=65.6.3=py39h06a4308_0 - - six=1.16.0=pyhd3eb1b0_1 - - sqlite=3.41.1=h5eee18b_0 - - tk=8.6.12=h1ccaba5_0 - - torchaudio=0.12.1=py39_cu113 - - torchvision=0.13.1=py39_cu113 - - typing_extensions=4.4.0=py39h06a4308_0 - - urllib3=1.26.15=py39h06a4308_0 - - wheel=0.38.4=py39h06a4308_0 - - xz=5.2.10=h5eee18b_1 - - zlib=1.2.13=h5eee18b_0 - - zstd=1.5.4=hc292b87_0 - - pip: - - accelerate==0.20.3 - - aiofiles==23.1.0 - - aiohttp==3.8.4 - - aiosignal==1.3.1 - - altair==4.2.2 - - antlr4-python3-runtime==4.9.3 - - anyio==3.6.2 - - appdirs==1.4.4 - - argon2-cffi==21.3.0 - - argon2-cffi-bindings==21.2.0 - - arrow==1.2.3 - - asttokens==2.2.1 - - async-timeout==4.0.2 - - attrs==22.2.0 - - backcall==0.2.0 - - beautifulsoup4==4.12.2 - - bitsandbytes==0.37.0 - - bleach==6.0.0 - - blis==0.7.9 - - braceexpand==0.1.7 - - catalogue==2.0.8 - - cchardet==2.1.7 - - chardet==5.1.0 - - click==8.1.3 - - cloudpickle==2.2.1 - - cmake==3.26.1 - - comm==0.1.3 - - confection==0.0.4 - - contourpy==1.0.7 - - cycler==0.11.0 - - cymem==2.0.7 - - datasets==2.13.1 - - debugpy==1.6.7 - - decorator==5.1.1 - - decord==0.6.0 - - defusedxml==0.7.1 - - dill==0.3.6 - - docker-pycreds==0.4.0 - - einops==0.6.0 - - entrypoints==0.4 - - executing==1.2.0 - - fairscale==0.4.13 - - fastapi==0.95.0 - - fastjsonschema==2.16.3 - - ffmpy==0.3.0 - - filelock==3.9.0 - - fire==0.5.0 - - fonttools==4.38.0 - - fqdn==1.5.1 - - frozenlist==1.3.3 - - fsspec==2023.4.0 - - gitdb==4.0.10 - - gitpython==3.1.31 - - gradio==3.24.1 - - gradio-client==0.0.8 - - h11==0.14.0 - - httpcore==0.16.3 - - httpx==0.23.3 - - huggingface-hub==0.16.4 - - imageio==2.31.1 - - importlib-metadata==6.2.0 - - importlib-resources==5.12.0 - - iopath==0.1.10 - - ipykernel==6.22.0 - - ipython==8.12.0 - - ipython-genutils==0.2.0 - - isoduration==20.11.0 - - jedi==0.18.2 - - jinja2==3.1.2 - - joblib==1.2.0 - - jsonpointer==2.3 - - jsonschema==4.17.3 - - jupyter-client==8.1.0 - - jupyter-core==5.3.0 - - jupyter-events==0.6.3 - - jupyter-server==2.5.0 - - jupyter-server-terminals==0.4.4 - - jupyterlab-pygments==0.2.2 - - kiwisolver==1.4.4 - - langcodes==3.3.0 - - lazy-loader==0.2 - - linkify-it-py==2.0.0 - - lit==16.0.0 - - llvmlite==0.39.1 - - loralib==0.1.1 - - markdown-it-py==2.2.0 - - markupsafe==2.1.2 - - matplotlib==3.7.0 - - matplotlib-inline==0.1.6 - - mdit-py-plugins==0.3.3 - - mdurl==0.1.2 - - mistune==2.0.5 - - mpmath==1.3.0 - - multidict==6.0.4 - - multiprocess==0.70.14 - - murmurhash==1.0.9 - - nbclassic==0.5.5 - - nbclient==0.7.3 - - nbconvert==7.3.0 - - nbformat==5.8.0 - - nest-asyncio==1.5.6 - - networkx==3.1 - - nltk==3.8.1 - - notebook==6.5.4 - - notebook-shim==0.2.2 - - numba==0.56.4 - - nvidia-cublas-cu11==11.10.3.66 - - nvidia-cuda-cupti-cu11==11.7.101 - - nvidia-cuda-nvrtc-cu11==11.7.99 - - nvidia-cuda-runtime-cu11==11.7.99 - - nvidia-cudnn-cu11==8.5.0.96 - - nvidia-cufft-cu11==10.9.0.58 - - nvidia-curand-cu11==10.2.10.91 - - nvidia-cusolver-cu11==11.4.0.1 - - nvidia-cusparse-cu11==11.7.4.91 - - nvidia-nccl-cu11==2.14.3 - - nvidia-nvtx-cu11==11.7.91 - - omegaconf==2.3.0 - - openai==0.27.0 - - opencv-python==4.7.0.72 - - orjson==3.8.9 - - packaging==23.0 - - pandas==2.0.0 - - pandocfilters==1.5.0 - - parso==0.8.3 - - pathlib==1.0.1 - - pathtools==0.1.2 - - pathy==0.10.1 - - peft==0.2.0 - - pexpect==4.8.0 - - pickleshare==0.7.5 - - platformdirs==3.2.0 - - policies==0.4.2 - - portalocker==2.7.0 - - preshed==3.0.8 - - progressbar2==4.2.0 - - prometheus-client==0.16.0 - - prompt-toolkit==3.0.38 - - protobuf==4.22.1 - - psutil==5.9.4 - - ptyprocess==0.7.0 - - pure-eval==0.2.2 - - pyarrow==12.0.1 - - pycocoevalcap==1.2 - - pycocotools==2.0.6 - - pycryptodome==3.17 - - pydantic==1.10.7 - - pydub==0.25.1 - - pygments==2.14.0 - - pynndescent==0.5.8 - - pyparsing==3.0.9 - - pyrsistent==0.19.3 - - python-dateutil==2.8.2 - - python-json-logger==2.0.7 - - python-multipart==0.0.6 - - python-utils==3.7.0 - - pytz==2023.3 - - pywavelets==1.4.1 - - pyyaml==6.0 - - pyzmq==25.0.2 - - regex==2022.10.31 - - rfc3339-validator==0.1.4 - - rfc3986==1.5.0 - - rfc3986-validator==0.1.1 - - safetensors==0.3.1 - - scikit-image==0.21.0 - - scikit-learn==1.2.2 - - scipy==1.10.1 - - seaborn==0.12.2 - - semantic-version==2.10.0 - - send2trash==1.8.0 - - sentence-transformers==2.2.2 - - sentencepiece==0.1.97 - - sentry-sdk==1.19.1 - - setproctitle==1.3.2 - - smart-open==6.3.0 - - smmap==5.0.0 - - sniffio==1.3.0 - - soupsieve==2.4 - - spacy==3.5.1 - - spacy-legacy==3.0.12 - - spacy-loggers==1.0.4 - - srsly==2.4.6 - - stack-data==0.6.2 - - starlette==0.26.1 - - submitit==1.4.5 - - sympy==1.11.1 - - tenacity==8.2.2 - - termcolor==2.3.0 - - terminado==0.17.1 - - thinc==8.1.9 - - threadpoolctl==3.1.0 - - tifffile==2023.4.12 - - timm==0.6.13 - - tinycss2==1.2.1 - - tokenizers==0.13.3 - - toolz==0.12.0 - - torch==2.0.0 - - tornado==6.2 - - tqdm==4.64.1 - - traitlets==5.9.0 - - transformers==4.31.0 - - triton==2.0.0 - - typer==0.7.0 - - tzdata==2023.3 - - uc-micro-py==1.0.1 - - umap-learn==0.5.3 - - uri-template==1.2.0 - - uvicorn==0.21.1 - - visual-genome==1.1.1 - - wandb==0.14.2 - - wasabi==1.1.1 - - wcwidth==0.2.6 - - webcolors==1.13 - - webdataset==0.2.48 - - webencodings==0.5.1 - - websocket-client==1.5.1 - - websockets==11.0.1 - - xxhash==3.2.0 - - yarl==1.8.2 - - zipp==3.14.0 -prefix: /home/chenj0g/anaconda3/envs/eye diff --git a/wandb/run-20231023_090058-141dzr9c/files/config.yaml b/wandb/run-20231023_090058-141dzr9c/files/config.yaml deleted file mode 100644 index 5c53ff8..0000000 --- a/wandb/run-20231023_090058-141dzr9c/files/config.yaml +++ /dev/null @@ -1,38 +0,0 @@ -wandb_version: 1 - -_wandb: - desc: null - value: - python_version: 3.9.16 - cli_version: 0.14.2 - framework: huggingface - huggingface_version: 4.31.0 - is_jupyter_run: false - is_kaggle_kernel: false - start_time: 1698040858.125822 - t: - 1: - - 1 - - 11 - - 41 - - 49 - - 55 - - 63 - - 71 - 2: - - 1 - - 11 - - 41 - - 49 - - 55 - - 63 - - 71 - 3: - - 1 - - 13 - - 23 - 4: 3.9.16 - 5: 0.14.2 - 6: 4.31.0 - 8: - - 5 diff --git a/wandb/run-20231023_090058-141dzr9c/files/output.log b/wandb/run-20231023_090058-141dzr9c/files/output.log deleted file mode 100644 index 70f57c3..0000000 --- a/wandb/run-20231023_090058-141dzr9c/files/output.log +++ /dev/null @@ -1,164 +0,0 @@ - -2023-10-23 09:01:04,885 [INFO] Start training -batch sizes [[2]] -module.llama_model.base_model.model.model.layers.0.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.0.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.0.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.0.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.1.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.1.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.1.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.1.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.2.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.2.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.2.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.2.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.3.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.3.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.3.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.3.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.4.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.4.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.4.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.4.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.5.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.5.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.5.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.5.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.6.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.6.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.6.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.6.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.7.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.7.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.7.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.7.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.8.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.8.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.8.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.8.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.9.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.9.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.9.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.9.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.10.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.10.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.10.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.10.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.11.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.11.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.11.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.11.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.12.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.12.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.12.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.12.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.13.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.13.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.13.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.13.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.14.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.14.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.14.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.14.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.15.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.15.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.15.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.15.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.16.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.16.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.16.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.16.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.17.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.17.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.17.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.17.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.18.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.18.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.18.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.18.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.19.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.19.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.19.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.19.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.20.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.20.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.20.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.20.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.21.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.21.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.21.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.21.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.22.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.22.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.22.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.22.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.23.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.23.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.23.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.23.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.24.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.24.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.24.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.24.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.25.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.25.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.25.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.25.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.26.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.26.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.26.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.26.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.27.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.27.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.27.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.27.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.28.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.28.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.28.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.28.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.29.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.29.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.29.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.29.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.30.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.30.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.30.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.30.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.31.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.31.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.31.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.31.self_attn.v_proj.lora_B.weight -module.llama_proj.weight -module.llama_proj.bias -2023-10-23 09:01:07,002 [INFO] dataset_ratios not specified, datasets will be concatenated (map-style datasets) or chained (webdataset.DataPipeline). -2023-10-23 09:01:07,002 [INFO] Loaded 12171 records for train split from the dataset. -2023-10-23 09:01:07,020 [INFO] number of trainable parameters: 56627200 -2023-10-23 09:01:07,021 [INFO] Start training epoch 0, 1000 iters per inner epoch. -Train: data epoch: [0] [ 0/1000] eta: 0:45:28 lr: 0.000001 loss: 1.3049 time: 2.7288 data: 0.0000 max mem: 33055 -/home/chenj0g/anaconda3/envs/eye/lib/python3.9/site-packages/torch/utils/checkpoint.py:31: UserWarning: None of the inputs have requires_grad=True. Gradients will be None - warnings.warn("None of the inputs have requires_grad=True. Gradients will be None") -Train: data epoch: [0] [ 50/1000] eta: 0:07:58 lr: 0.000001 loss: 1.2327 time: 0.4508 data: 0.0000 max mem: 38295 -Train: data epoch: [0] [ 100/1000] eta: 0:07:10 lr: 0.000002 loss: 1.1049 time: 0.4457 data: 0.0000 max mem: 39575 -Train: data epoch: [0] [ 150/1000] eta: 0:06:42 lr: 0.000002 loss: 1.2879 time: 0.4630 data: 0.0000 max mem: 40211 -Train: data epoch: [0] [ 200/1000] eta: 0:06:18 lr: 0.000003 loss: 1.0077 time: 0.4527 data: 0.0000 max mem: 40211 -Train: data epoch: [0] [ 250/1000] eta: 0:05:52 lr: 0.000003 loss: 1.4790 time: 0.4626 data: 0.0000 max mem: 41450 -Train: data epoch: [0] [ 300/1000] eta: 0:05:27 lr: 0.000004 loss: 1.2864 time: 0.4639 data: 0.0000 max mem: 41450 -Traceback (most recent call last): - File "/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/train.py", line 118, in - main() - File "/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/train.py", line 114, in main - runner.train() - File "/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/minigpt4/runners/runner_base.py", line 377, in train - train_stats = self.train_epoch(cur_epoch) - File "/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/minigpt4/runners/runner_base.py", line 437, in train_epoch - return self.task.train_epoch( - File "/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/minigpt4/tasks/base_task.py", line 116, in train_epoch - return self._train_inner_loop( - File "/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/minigpt4/tasks/base_task.py", line 225, in _train_inner_loop - scaler.scale(loss).backward() - File "/home/chenj0g/anaconda3/envs/eye/lib/python3.9/site-packages/torch/_tensor.py", line 487, in backward - torch.autograd.backward( - File "/home/chenj0g/anaconda3/envs/eye/lib/python3.9/site-packages/torch/autograd/__init__.py", line 200, in backward - Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass -KeyboardInterrupt \ No newline at end of file diff --git a/wandb/run-20231023_090058-141dzr9c/files/requirements.txt b/wandb/run-20231023_090058-141dzr9c/files/requirements.txt deleted file mode 100644 index 75bc393..0000000 --- a/wandb/run-20231023_090058-141dzr9c/files/requirements.txt +++ /dev/null @@ -1,246 +0,0 @@ -accelerate==0.20.3 -aiofiles==23.2.1 -aiohttp==3.8.4 -aiosignal==1.3.1 -altair==5.1.2 -antlr4-python3-runtime==4.9.3 -anyio==3.6.2 -appdirs==1.4.4 -argon2-cffi-bindings==21.2.0 -argon2-cffi==21.3.0 -arrow==1.2.3 -asttokens==2.2.1 -async-timeout==4.0.2 -attrs==22.2.0 -backcall==0.2.0 -beautifulsoup4==4.12.2 -bitsandbytes==0.37.0 -bleach==6.0.0 -blis==0.7.9 -braceexpand==0.1.7 -brotlipy==0.7.0 -catalogue==2.0.8 -cchardet==2.1.7 -certifi==2022.12.7 -cffi==1.15.1 -chardet==5.1.0 -charset-normalizer==2.0.4 -click==8.1.3 -cloudpickle==2.2.1 -cmake==3.26.1 -comm==0.1.3 -confection==0.0.4 -contourpy==1.0.7 -cryptography==39.0.1 -cycler==0.11.0 -cymem==2.0.7 -datasets==2.13.1 -debugpy==1.6.7 -decorator==5.1.1 -decord==0.6.0 -defusedxml==0.7.1 -dill==0.3.6 -docker-pycreds==0.4.0 -einops==0.6.0 -entrypoints==0.4 -executing==1.2.0 -fairscale==0.4.13 -fastapi==0.95.0 -fastjsonschema==2.16.3 -ffmpy==0.3.1 -filelock==3.9.0 -fire==0.5.0 -flit-core==3.8.0 -fonttools==4.38.0 -fqdn==1.5.1 -frozenlist==1.3.3 -fsspec==2023.9.2 -gitdb==4.0.10 -gitpython==3.1.31 -gradio-client==0.6.0 -gradio==3.24.1 -h11==0.14.0 -httpcore==0.18.0 -httpx==0.25.0 -huggingface-hub==0.18.0 -idna==3.4 -imageio==2.31.1 -importlib-metadata==6.2.0 -importlib-resources==5.12.0 -iopath==0.1.10 -ipykernel==6.22.0 -ipython-genutils==0.2.0 -ipython==8.12.0 -isoduration==20.11.0 -jedi==0.18.2 -jinja2==3.1.2 -joblib==1.2.0 -jsonpointer==2.3 -jsonschema==4.17.3 -jupyter-client==8.1.0 -jupyter-core==5.3.0 -jupyter-events==0.6.3 -jupyter-server-terminals==0.4.4 -jupyter-server==2.5.0 -jupyterlab-pygments==0.2.2 -kiwisolver==1.4.4 -langcodes==3.3.0 -lazy-loader==0.2 -linkify-it-py==2.0.2 -lit==16.0.0 -llvmlite==0.39.1 -loralib==0.1.1 -markdown-it-py==2.2.0 -markupsafe==2.1.2 -matplotlib-inline==0.1.6 -matplotlib==3.7.0 -mdit-py-plugins==0.3.3 -mdurl==0.1.2 -mistune==2.0.5 -mkl-fft==1.3.1 -mkl-random==1.2.2 -mkl-service==2.4.0 -mpmath==1.3.0 -multidict==6.0.4 -multiprocess==0.70.14 -murmurhash==1.0.9 -nbclassic==0.5.5 -nbclient==0.7.3 -nbconvert==7.3.0 -nbformat==5.8.0 -nest-asyncio==1.5.6 -networkx==3.1 -nltk==3.8.1 -notebook-shim==0.2.2 -notebook==6.5.4 -numba==0.56.4 -numpy==1.23.5 -nvidia-cublas-cu11==11.10.3.66 -nvidia-cuda-cupti-cu11==11.7.101 -nvidia-cuda-nvrtc-cu11==11.7.99 -nvidia-cuda-runtime-cu11==11.7.99 -nvidia-cudnn-cu11==8.5.0.96 -nvidia-cufft-cu11==10.9.0.58 -nvidia-curand-cu11==10.2.10.91 -nvidia-cusolver-cu11==11.4.0.1 -nvidia-cusparse-cu11==11.7.4.91 -nvidia-nccl-cu11==2.14.3 -nvidia-nvtx-cu11==11.7.91 -omegaconf==2.3.0 -openai==0.27.0 -opencv-python==4.7.0.72 -orjson==3.9.8 -packaging==23.0 -pandas==2.0.0 -pandocfilters==1.5.0 -parso==0.8.3 -pathlib==1.0.1 -pathtools==0.1.2 -pathy==0.10.1 -peft==0.2.0 -pexpect==4.8.0 -pickleshare==0.7.5 -pillow==9.4.0 -pip==23.0.1 -platformdirs==3.2.0 -policies==0.4.2 -portalocker==2.7.0 -preshed==3.0.8 -progressbar2==4.2.0 -prometheus-client==0.16.0 -prompt-toolkit==3.0.38 -protobuf==4.22.1 -psutil==5.9.4 -ptyprocess==0.7.0 -pure-eval==0.2.2 -pyarrow==12.0.1 -pycocoevalcap==1.2 -pycocotools==2.0.6 -pycparser==2.21 -pycryptodome==3.17 -pydantic==1.10.7 -pydub==0.25.1 -pygments==2.14.0 -pynndescent==0.5.8 -pyopenssl==23.0.0 -pyparsing==3.0.9 -pyrsistent==0.19.3 -pysocks==1.7.1 -python-dateutil==2.8.2 -python-json-logger==2.0.7 -python-multipart==0.0.6 -python-utils==3.7.0 -pytz==2023.3 -pywavelets==1.4.1 -pyyaml==6.0 -pyzmq==25.0.2 -regex==2022.10.31 -requests==2.28.1 -rfc3339-validator==0.1.4 -rfc3986-validator==0.1.1 -rfc3986==1.5.0 -rpe-index==1.2.0 -safetensors==0.3.1 -scikit-image==0.21.0 -scikit-learn==1.2.2 -scipy==1.10.1 -seaborn==0.12.2 -semantic-version==2.10.0 -send2trash==1.8.0 -sentence-transformers==2.2.2 -sentencepiece==0.1.97 -sentry-sdk==1.19.1 -setproctitle==1.3.2 -setuptools==65.6.3 -six==1.16.0 -smart-open==6.3.0 -smmap==5.0.0 -sniffio==1.3.0 -soupsieve==2.4 -spacy-legacy==3.0.12 -spacy-loggers==1.0.4 -spacy==3.5.1 -srsly==2.4.6 -stack-data==0.6.2 -starlette==0.26.1 -submitit==1.4.5 -sympy==1.11.1 -tenacity==8.2.2 -termcolor==2.3.0 -terminado==0.17.1 -thinc==8.1.9 -threadpoolctl==3.1.0 -tifffile==2023.4.12 -timm==0.6.13 -tinycss2==1.2.1 -tokenizers==0.13.3 -toolz==0.12.0 -torch==2.0.0 -torchaudio==0.12.1 -torchvision==0.13.1 -tornado==6.2 -tqdm==4.64.1 -traitlets==5.9.0 -transformers==4.31.0 -triton==2.0.0 -typer==0.7.0 -typing-extensions==4.4.0 -tzdata==2023.3 -uc-micro-py==1.0.2 -umap-learn==0.5.3 -uri-template==1.2.0 -urllib3==1.26.15 -uvicorn==0.21.1 -visual-genome==1.1.1 -wandb==0.14.2 -wasabi==1.1.1 -wcwidth==0.2.6 -webcolors==1.13 -webdataset==0.2.48 -webencodings==0.5.1 -websocket-client==1.5.1 -websockets==11.0.1 -wheel==0.38.4 -xxhash==3.2.0 -yarl==1.8.2 -zipp==3.14.0 \ No newline at end of file diff --git a/wandb/run-20231023_090058-141dzr9c/files/wandb-metadata.json b/wandb/run-20231023_090058-141dzr9c/files/wandb-metadata.json deleted file mode 100644 index f7f0519..0000000 --- a/wandb/run-20231023_090058-141dzr9c/files/wandb-metadata.json +++ /dev/null @@ -1,374 +0,0 @@ -{ - "os": "Linux-3.10.0-1160.76.1.el7.x86_64-x86_64-with-glibc2.17", - "python": "3.9.16", - "heartbeatAt": "2023-10-23T06:00:59.460778", - "startedAt": "2023-10-23T06:00:58.084251", - "docker": null, - "cuda": null, - "args": [ - "--job_name", - "minigpt4_v2_test", - "--cfg-path", - "train_configs/minigpt_v2_finetune.yaml" - ], - "state": "running", - "program": "/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/train.py", - "codePath": "train.py", - "git": { - "remote": "https://github.com/junchen14/MiniGPT-4_finetune", - "commit": "5c3ec8bb730b2e9df23b4bb3eb24108f9c04a8d6" - }, - "email": "jun.chen@kaust.edu.sa", - "root": "/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune", - "host": "gpu201-16-l", - "username": "chenj0g", - "executable": "/home/chenj0g/anaconda3/envs/eye/bin/python", - "cpu_count": 64, - "cpu_count_logical": 64, - "cpu_freq": { - "current": 1996.2770000000016, - "min": 0.0, - "max": 0.0 - }, - "cpu_freq_per_core": [ - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - } - ], - "disk": { - "total": 251.85773849487305, - "used": 7.894733428955078 - }, - "gpu": "NVIDIA A100-SXM4-80GB", - "gpu_count": 2, - "gpu_devices": [ - { - "name": "NVIDIA A100-SXM4-80GB", - "memory_total": 85899345920 - }, - { - "name": "NVIDIA A100-SXM4-80GB", - "memory_total": 85899345920 - } - ], - "memory": { - "total": 503.7154769897461 - } -} diff --git a/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json b/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json deleted file mode 100644 index f3fca1a..0000000 --- a/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json +++ /dev/null @@ -1 +0,0 @@ -{"epoch": 0, "loss": 1.0476144552230835, "_timestamp": 1698041023.1460156, "_runtime": 165.02019357681274, "_step": 334, "_wandb": {"runtime": 164}} \ No newline at end of file diff --git a/wandb/run-20231023_090058-141dzr9c/logs/debug-internal.log b/wandb/run-20231023_090058-141dzr9c/logs/debug-internal.log deleted file mode 100644 index d524e8d..0000000 --- a/wandb/run-20231023_090058-141dzr9c/logs/debug-internal.log +++ /dev/null @@ -1,1701 +0,0 @@ -2023-10-23 09:00:58,125 INFO StreamThr :51707 [internal.py:wandb_internal():86] W&B internal server running at pid: 51707, started at: 2023-10-23 09:00:58.125014 -2023-10-23 09:00:58,126 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status -2023-10-23 09:00:58,127 INFO WriterThread:51707 [datastore.py:open_for_write():85] open: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/run-141dzr9c.wandb -2023-10-23 09:00:58,129 DEBUG SenderThread:51707 [sender.py:send():375] send: header -2023-10-23 09:00:58,534 DEBUG SenderThread:51707 [sender.py:send():375] send: run -2023-10-23 09:00:59,117 INFO SenderThread:51707 [dir_watcher.py:__init__():219] watching files in: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files -2023-10-23 09:00:59,117 INFO SenderThread:51707 [sender.py:_start_run_threads():1124] run started: 141dzr9c with start time 1698040858.125822 -2023-10-23 09:00:59,117 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:00:59,118 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:00:59,119 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: check_version -2023-10-23 09:00:59,119 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: check_version -2023-10-23 09:00:59,439 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: run_start -2023-10-23 09:00:59,444 DEBUG HandlerThread:51707 [system_info.py:__init__():31] System info init -2023-10-23 09:00:59,444 DEBUG HandlerThread:51707 [system_info.py:__init__():46] System info init done -2023-10-23 09:00:59,444 INFO HandlerThread:51707 [system_monitor.py:start():181] Starting system monitor -2023-10-23 09:00:59,444 INFO SystemMonitor:51707 [system_monitor.py:_start():145] Starting system asset monitoring threads -2023-10-23 09:00:59,444 INFO HandlerThread:51707 [system_monitor.py:probe():201] Collecting system info -2023-10-23 09:00:59,445 INFO SystemMonitor:51707 [interfaces.py:start():190] Started cpu monitoring -2023-10-23 09:00:59,445 INFO SystemMonitor:51707 [interfaces.py:start():190] Started disk monitoring -2023-10-23 09:00:59,447 INFO SystemMonitor:51707 [interfaces.py:start():190] Started gpu monitoring -2023-10-23 09:00:59,447 INFO SystemMonitor:51707 [interfaces.py:start():190] Started memory monitoring -2023-10-23 09:00:59,447 INFO SystemMonitor:51707 [interfaces.py:start():190] Started network monitoring -2023-10-23 09:00:59,460 DEBUG HandlerThread:51707 [system_info.py:probe():195] Probing system -2023-10-23 09:00:59,466 DEBUG HandlerThread:51707 [system_info.py:_probe_git():180] Probing git -2023-10-23 09:00:59,479 DEBUG HandlerThread:51707 [system_info.py:_probe_git():188] Probing git done -2023-10-23 09:00:59,479 DEBUG HandlerThread:51707 [system_info.py:probe():240] Probing system done -2023-10-23 09:00:59,479 DEBUG HandlerThread:51707 [system_monitor.py:probe():210] {'os': 'Linux-3.10.0-1160.76.1.el7.x86_64-x86_64-with-glibc2.17', 'python': '3.9.16', 'heartbeatAt': '2023-10-23T06:00:59.460778', 'startedAt': '2023-10-23T06:00:58.084251', 'docker': None, 'cuda': None, 'args': ('--job_name', 'minigpt4_v2_test', '--cfg-path', 'train_configs/minigpt_v2_finetune.yaml'), 'state': 'running', 'program': '/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/train.py', 'codePath': 'train.py', 'git': {'remote': 'https://github.com/junchen14/MiniGPT-4_finetune', 'commit': '5c3ec8bb730b2e9df23b4bb3eb24108f9c04a8d6'}, 'email': 'jun.chen@kaust.edu.sa', 'root': '/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune', 'host': 'gpu201-16-l', 'username': 'chenj0g', 'executable': '/home/chenj0g/anaconda3/envs/eye/bin/python', 'cpu_count': 64, 'cpu_count_logical': 64, 'cpu_freq': {'current': 1996.2770000000016, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}], 'disk': {'total': 251.85773849487305, 'used': 7.894733428955078}, 'gpu': 'NVIDIA A100-SXM4-80GB', 'gpu_count': 2, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}], 'memory': {'total': 503.7154769897461}} -2023-10-23 09:00:59,479 INFO HandlerThread:51707 [system_monitor.py:probe():211] Finished collecting system info -2023-10-23 09:00:59,479 INFO HandlerThread:51707 [system_monitor.py:probe():214] Publishing system info -2023-10-23 09:00:59,479 DEBUG HandlerThread:51707 [system_info.py:_save_pip():51] Saving list of pip packages installed into the current environment -2023-10-23 09:00:59,480 DEBUG HandlerThread:51707 [system_info.py:_save_pip():67] Saving pip packages done -2023-10-23 09:00:59,480 DEBUG HandlerThread:51707 [system_info.py:_save_conda():74] Saving list of conda packages installed into the current environment -2023-10-23 09:01:00,119 INFO Thread-12 :51707 [dir_watcher.py:_on_file_created():278] file/dir created: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:00,119 INFO Thread-12 :51707 [dir_watcher.py:_on_file_created():278] file/dir created: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/requirements.txt -2023-10-23 09:01:00,119 INFO Thread-12 :51707 [dir_watcher.py:_on_file_created():278] file/dir created: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/conda-environment.yaml -2023-10-23 09:01:04,866 DEBUG HandlerThread:51707 [system_info.py:_save_conda():86] Saving conda packages done -2023-10-23 09:01:04,867 INFO HandlerThread:51707 [system_monitor.py:probe():216] Finished publishing system info -2023-10-23 09:01:04,873 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:01:04,873 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: keepalive -2023-10-23 09:01:04,873 DEBUG SenderThread:51707 [sender.py:send():375] send: files -2023-10-23 09:01:04,873 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-metadata.json with policy now -2023-10-23 09:01:04,879 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 09:01:04,879 DEBUG SenderThread:51707 [sender.py:send():375] send: telemetry -2023-10-23 09:01:04,879 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: stop_status -2023-10-23 09:01:05,122 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/conda-environment.yaml -2023-10-23 09:01:05,122 INFO Thread-12 :51707 [dir_watcher.py:_on_file_created():278] file/dir created: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-metadata.json -2023-10-23 09:01:05,281 DEBUG SenderThread:51707 [sender.py:send():375] send: telemetry -2023-10-23 09:01:05,669 INFO wandb-upload_0:51707 [upload_job.py:push():137] Uploaded file /tmp/tmpkvnc73e8wandb/uumvtbka-wandb-metadata.json -2023-10-23 09:01:06,122 INFO Thread-12 :51707 [dir_watcher.py:_on_file_created():278] file/dir created: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/output.log -2023-10-23 09:01:08,124 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/output.log -2023-10-23 09:01:09,063 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:01:09,124 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/output.log -2023-10-23 09:01:09,750 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:09,751 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:09,751 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:09,752 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:10,125 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:10,125 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/output.log -2023-10-23 09:01:10,187 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:10,187 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:10,187 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:10,188 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:10,698 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:10,698 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:10,698 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:10,699 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:11,091 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:11,092 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:11,092 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:11,092 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:11,126 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:11,126 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/output.log -2023-10-23 09:01:11,522 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:11,523 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:11,523 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:11,523 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:12,105 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:12,105 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:12,106 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:12,106 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:12,127 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:12,127 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/output.log -2023-10-23 09:01:12,546 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:12,546 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:12,546 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:12,547 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:13,004 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:13,005 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:13,005 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:13,005 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:13,127 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:13,503 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:13,504 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:13,504 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:13,506 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:13,931 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:13,932 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:13,932 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:13,933 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:14,128 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:14,378 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:14,378 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:14,378 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:14,379 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:01:14,379 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:14,841 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:14,841 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:14,842 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:14,842 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:15,129 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:15,222 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:15,222 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:15,223 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:15,223 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:15,728 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:15,728 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:15,729 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:15,731 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:16,129 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:16,151 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:16,151 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:16,151 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:16,152 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:16,692 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:16,692 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:16,693 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:16,693 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:17,064 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:17,064 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:17,065 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:17,065 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:17,130 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:17,618 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:17,618 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:17,619 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:17,619 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:18,119 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:18,120 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:18,120 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:18,122 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:18,131 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:18,572 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:18,573 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:18,573 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:18,574 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:19,001 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:19,001 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:19,002 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:19,002 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:19,132 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:19,451 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:19,451 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:19,452 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:19,452 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:01:19,452 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:19,878 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 09:01:19,879 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: stop_status -2023-10-23 09:01:19,991 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:20,096 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:20,096 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:20,097 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:20,133 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:20,454 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:20,455 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:20,455 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:20,457 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:20,896 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:20,896 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:20,896 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:20,897 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:21,134 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:21,361 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:21,361 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:21,362 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:21,363 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:21,788 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:21,789 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:21,789 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:21,790 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:22,134 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:22,240 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:22,240 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:22,241 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:22,241 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:22,703 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:22,704 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:22,704 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:22,706 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:23,135 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:23,142 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:23,142 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:23,142 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:23,143 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:23,695 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:23,695 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:23,695 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:23,696 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:24,136 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:24,141 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:24,141 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:24,141 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:24,142 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:24,682 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:24,683 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:24,683 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:24,683 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:01:24,684 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:25,067 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:25,067 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:25,067 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:25,069 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:25,136 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:25,489 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:25,490 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:25,490 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:25,491 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:25,873 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:25,873 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:25,873 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:25,874 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:26,137 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:26,324 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:26,325 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:26,325 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:26,326 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:26,779 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:26,779 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:26,779 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:26,780 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:27,138 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:27,257 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:27,257 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:27,257 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:27,259 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:27,730 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:27,731 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:27,731 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:27,732 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:28,139 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:28,166 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:28,167 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:28,167 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:28,168 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:28,552 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:28,553 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:28,553 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:28,553 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:29,033 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:29,034 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:29,034 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:29,034 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:29,139 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:29,478 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:29,478 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:29,478 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:29,480 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:29,914 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:29,914 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:29,918 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:01:30,125 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:30,125 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:30,140 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:30,140 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/config.yaml -2023-10-23 09:01:30,425 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:30,426 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:30,426 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:30,426 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:30,825 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:30,826 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:30,826 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:30,827 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:31,141 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:31,265 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:31,266 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:31,266 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:31,267 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:31,770 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:31,770 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:31,770 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:31,772 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:32,141 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:32,213 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:32,214 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:32,214 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:32,215 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:32,711 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:32,712 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:32,712 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:32,713 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:33,142 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:33,158 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:33,158 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:33,159 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:33,159 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:33,582 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:33,583 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:33,583 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:33,584 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:34,013 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:34,013 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:34,013 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:34,019 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:34,143 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:34,457 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:34,457 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:34,458 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:34,458 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:34,878 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 09:01:34,879 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: stop_status -2023-10-23 09:01:34,944 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:35,104 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:35,104 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:35,110 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:35,144 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:35,144 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/output.log -2023-10-23 09:01:35,380 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:35,381 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:35,381 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:35,381 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:01:35,382 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:35,803 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:35,804 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:35,804 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:35,805 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:36,144 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:36,268 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:36,269 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:36,269 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:36,271 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:36,633 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:36,634 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:36,634 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:36,635 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:37,094 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:37,095 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:37,095 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:37,096 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:37,145 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:37,585 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:37,586 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:37,586 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:37,587 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:38,027 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:38,027 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:38,027 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:38,028 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:38,146 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:38,576 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:38,577 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:38,577 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:38,579 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:39,051 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:39,052 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:39,052 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:39,053 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:39,146 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:39,497 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:39,497 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:39,498 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:39,499 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:39,951 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:39,951 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:39,952 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:39,952 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:40,147 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:40,372 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:40,373 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:40,373 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:40,373 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:40,816 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:40,817 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:40,817 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:40,818 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:01:40,819 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:41,148 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:41,383 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:41,384 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:41,384 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:41,385 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:41,829 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:41,829 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:41,830 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:41,830 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:42,148 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:42,290 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:42,290 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:42,290 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:42,291 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:42,807 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:42,807 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:42,807 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:42,808 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:43,149 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:43,228 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:43,229 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:43,229 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:43,231 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:43,729 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:43,730 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:43,730 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:43,731 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:44,150 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:44,201 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:44,201 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:44,201 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:44,202 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:44,673 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:44,674 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:44,674 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:44,675 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:45,105 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:45,106 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:45,106 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:45,109 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:45,151 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:45,561 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:45,562 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:45,562 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:45,564 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:45,948 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:45,948 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:45,949 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:45,949 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:01:45,949 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:46,153 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:46,422 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:46,423 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:46,423 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:46,424 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:46,893 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:46,893 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:46,893 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:46,894 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:47,154 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:47,396 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:47,397 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:47,397 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:47,398 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:47,821 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:47,821 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:47,821 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:47,823 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:48,155 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:48,189 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:48,189 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:48,189 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:48,190 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:48,615 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:48,616 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:48,616 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:48,617 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:49,085 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:49,086 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:49,086 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:49,087 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:49,156 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:49,534 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:49,535 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:49,535 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:49,536 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:49,878 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 09:01:49,879 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: stop_status -2023-10-23 09:01:50,048 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:50,156 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:50,168 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:50,168 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:50,170 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:50,412 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:50,412 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:50,412 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:50,413 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:50,805 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:50,805 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:50,805 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:50,806 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:51,157 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:51,282 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:51,282 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:51,283 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:51,283 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:01:51,283 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:51,720 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:51,721 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:51,721 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:51,722 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:52,158 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:52,193 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:52,193 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:52,193 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:52,195 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:52,626 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:52,627 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:52,627 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:52,627 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:53,049 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:53,050 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:53,050 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:53,051 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:53,159 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:53,446 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:53,446 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:53,446 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:53,447 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:53,881 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:53,881 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:53,881 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:53,882 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:54,164 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:54,493 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:54,493 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:54,494 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:54,496 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:54,936 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:54,936 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:54,937 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:54,937 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:55,165 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:55,336 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:55,337 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:55,337 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:55,338 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:55,878 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:55,879 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:55,879 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:55,880 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:56,165 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:56,369 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:56,369 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:56,369 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:56,369 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:01:56,370 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:56,753 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:56,754 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:56,754 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:56,756 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:57,166 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:57,166 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/output.log -2023-10-23 09:01:57,212 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:57,213 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:57,213 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:57,214 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:57,688 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:57,689 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:57,689 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:57,690 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:58,167 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:58,274 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:58,274 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:58,275 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:58,275 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:58,844 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:58,844 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:58,845 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:58,845 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:59,167 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:01:59,305 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:59,305 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:59,305 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:59,306 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:01:59,448 DEBUG SystemMonitor:51707 [system_monitor.py:_start():159] Starting system metrics aggregation loop -2023-10-23 09:01:59,450 DEBUG SenderThread:51707 [sender.py:send():375] send: stats -2023-10-23 09:01:59,720 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:01:59,721 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:01:59,721 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:01:59,721 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:00,168 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:00,187 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:00,188 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:00,188 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:00,189 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:00,665 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:00,666 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:00,666 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:00,667 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:01,080 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:01,081 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:01,081 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:01,082 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:01,169 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:01,546 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:01,546 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:01,546 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:01,547 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:02:01,547 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:01,943 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:01,944 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:01,944 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:01,945 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:02,170 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:02,420 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:02,421 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:02,421 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:02,421 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:02,866 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:02,867 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:02,867 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:02,868 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:03,170 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:03,327 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:03,328 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:03,328 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:03,329 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:03,776 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:03,777 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:03,777 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:03,778 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:04,171 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:04,217 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:04,218 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:04,218 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:04,219 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:04,701 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:04,702 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:04,702 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:04,703 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:04,878 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 09:02:04,879 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: stop_status -2023-10-23 09:02:05,083 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:05,170 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:05,170 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:05,171 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:05,172 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:05,556 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:05,556 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:05,556 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:05,557 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:06,036 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:06,036 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:06,036 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:06,037 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:06,172 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:06,509 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:06,510 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:06,510 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:06,511 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:06,970 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:06,970 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:06,971 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:06,972 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:02:06,973 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:07,173 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:07,520 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:07,521 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:07,521 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:07,522 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:07,963 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:07,964 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:07,964 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:07,965 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:08,174 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:08,373 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:08,374 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:08,374 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:08,375 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:08,849 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:08,849 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:08,849 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:08,850 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:09,174 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:09,268 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:09,269 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:09,269 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:09,270 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:09,771 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:09,771 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:09,771 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:09,772 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:10,175 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:10,213 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:10,214 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:10,214 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:10,214 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:10,639 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:10,640 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:10,640 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:10,641 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:11,086 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:11,087 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:11,087 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:11,088 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:11,176 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:11,483 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:11,483 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:11,483 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:11,484 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:11,909 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:11,909 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:11,909 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:11,910 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:12,176 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:12,414 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:12,415 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:12,415 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:12,415 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:02:12,416 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:12,841 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:12,841 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:12,842 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:12,842 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:13,177 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:13,283 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:13,283 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:13,283 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:13,284 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:13,752 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:13,752 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:13,753 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:13,753 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:14,178 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:14,211 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:14,212 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:14,212 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:14,213 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:14,705 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:14,706 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:14,706 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:14,707 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:15,179 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:15,250 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:15,251 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:15,251 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:15,251 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:15,755 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:15,755 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:15,756 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:15,756 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:16,179 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:16,261 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:16,262 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:16,262 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:16,263 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:16,748 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:16,749 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:16,749 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:16,750 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:17,180 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:17,257 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:17,257 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:17,257 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:17,259 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:17,620 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:17,620 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:17,621 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:17,621 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:02:17,621 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:18,093 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:18,093 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:18,094 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:18,094 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:18,181 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:18,528 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:18,529 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:18,529 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:18,530 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:18,974 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:18,975 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:18,975 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:18,976 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:19,181 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:19,182 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/output.log -2023-10-23 09:02:19,491 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:19,492 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:19,492 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:19,493 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:19,878 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 09:02:19,879 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: stop_status -2023-10-23 09:02:20,048 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:20,116 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:20,117 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:20,117 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:20,182 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:20,485 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:20,485 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:20,486 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:20,486 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:20,879 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:20,880 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:20,880 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:20,881 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:21,183 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:21,424 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:21,425 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:21,425 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:21,426 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:21,980 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:21,981 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:21,981 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:21,982 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:22,184 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:22,594 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:22,595 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:22,595 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:22,596 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:23,051 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:23,052 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:23,052 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:23,052 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:02:23,053 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:23,184 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:23,555 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:23,555 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:23,555 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:23,556 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:24,099 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:24,100 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:24,100 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:24,101 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:24,185 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:24,483 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:24,483 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:24,483 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:24,484 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:24,950 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:24,951 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:24,951 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:24,952 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:25,186 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:25,410 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:25,411 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:25,411 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:25,412 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:25,927 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:25,928 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:25,928 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:25,929 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:26,186 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:26,480 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:26,481 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:26,481 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:26,482 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:26,879 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:26,879 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:26,879 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:26,880 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:27,187 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:27,320 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:27,320 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:27,320 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:27,322 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:27,779 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:27,779 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:27,780 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:27,780 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:28,188 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:28,298 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:28,298 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:28,299 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:28,299 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:02:28,299 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:28,793 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:28,794 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:28,794 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:28,794 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:29,177 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:29,178 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:29,178 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:29,179 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:29,188 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:29,451 DEBUG SenderThread:51707 [sender.py:send():375] send: stats -2023-10-23 09:02:29,639 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:29,639 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:29,639 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:29,640 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:30,115 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:30,116 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:30,116 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:30,117 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:30,189 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:30,593 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:30,594 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:30,594 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:30,595 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:31,077 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:31,077 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:31,078 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:31,078 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:31,190 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:31,539 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:31,539 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:31,539 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:31,540 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:32,003 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:32,003 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:32,003 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:32,004 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:32,191 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:32,448 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:32,449 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:32,449 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:32,449 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:32,947 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:32,948 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:32,948 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:32,949 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:33,191 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:33,454 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:33,455 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:33,455 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:33,455 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:02:33,456 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:33,901 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:33,901 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:33,901 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:33,902 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:34,192 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:34,371 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:34,371 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:34,372 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:34,372 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:34,811 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:34,812 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:34,812 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:34,813 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:34,879 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 09:02:34,879 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: stop_status -2023-10-23 09:02:35,193 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:35,204 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:35,205 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:35,205 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:35,206 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:35,635 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:35,635 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:35,635 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:35,636 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:36,088 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:36,089 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:36,089 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:36,090 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:36,193 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:36,467 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:36,468 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:36,468 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:36,469 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:36,840 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:36,841 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:36,841 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:36,842 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:37,194 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:37,306 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:37,306 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:37,307 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:37,307 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:37,643 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:37,643 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:37,644 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:37,646 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:38,195 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:38,217 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:38,218 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:38,218 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:38,219 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:38,714 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:38,715 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:38,715 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:38,715 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:02:38,716 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:39,095 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:39,095 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:39,096 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:39,096 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:39,196 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:39,560 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:39,561 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:39,561 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:39,562 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:40,054 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:40,055 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:40,055 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:40,056 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:40,196 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:40,491 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:40,492 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:40,492 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:40,493 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:41,037 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:41,038 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:41,038 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:41,039 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:41,197 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:41,483 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:41,484 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:41,484 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:41,485 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:42,001 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:42,002 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:42,002 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:42,003 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:42,198 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:42,463 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:42,463 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:42,463 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:42,465 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:42,895 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:42,896 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:42,896 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:42,897 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:43,198 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:43,198 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/output.log -2023-10-23 09:02:43,329 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:43,329 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:43,329 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:43,330 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:43,809 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:43,809 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:43,810 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:43,810 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:02:43,810 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:44,199 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:44,257 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:44,257 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:44,258 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:44,258 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:44,669 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:44,670 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:44,670 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:44,670 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:45,138 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:45,138 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:45,139 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:45,139 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:45,200 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:45,617 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:45,617 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:45,618 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:45,618 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:46,129 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:46,129 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:46,129 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:46,130 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:46,200 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:46,572 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:46,573 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:46,573 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:46,574 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:47,039 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:47,039 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:47,040 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:47,040 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:47,201 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:47,497 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:47,498 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:47,498 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:47,499 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:48,072 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:48,073 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:48,073 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:48,075 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:48,202 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:48,529 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:48,530 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:48,530 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:48,531 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:48,891 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:48,891 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:48,892 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:48,892 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:02:48,892 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:49,202 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:49,346 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:49,346 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:49,346 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:49,347 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:49,787 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:49,787 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:49,787 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:49,788 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:49,882 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 09:02:49,883 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: stop_status -2023-10-23 09:02:50,172 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:50,173 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:50,173 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:50,174 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:50,203 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:50,621 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:50,622 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:50,622 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:50,623 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:51,004 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:51,004 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:51,005 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:51,005 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:51,204 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:51,548 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:51,549 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:51,549 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:51,550 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:51,932 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:51,932 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:51,932 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:51,933 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:52,204 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:52,555 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:52,555 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:52,556 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:52,556 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:52,938 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:52,938 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:52,939 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:52,939 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:53,205 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:53,311 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:53,311 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:53,312 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:53,312 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:53,679 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:53,679 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:53,679 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:53,680 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:54,206 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:54,236 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:54,236 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:54,236 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:54,236 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:02:54,237 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:54,756 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:54,757 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:54,757 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:54,757 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:55,182 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:55,183 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:55,183 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:55,184 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:55,207 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:55,607 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:55,608 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:55,608 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:55,609 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:56,187 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:56,187 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:56,188 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:56,188 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:56,207 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:56,656 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:56,657 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:56,657 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:56,658 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:57,111 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:57,112 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:57,112 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:57,114 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:57,208 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:57,505 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:57,506 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:57,506 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:57,506 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:57,950 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:57,951 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:57,951 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:57,952 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:58,209 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:58,428 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:58,429 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:58,429 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:58,431 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:59,050 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:59,051 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:59,051 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:59,052 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:59,209 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:02:59,452 DEBUG SenderThread:51707 [sender.py:send():375] send: stats -2023-10-23 09:02:59,452 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:02:59,484 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:59,484 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:59,485 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:59,485 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:02:59,930 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:02:59,931 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:02:59,931 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:02:59,932 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:00,210 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:00,407 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:00,408 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:00,408 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:00,409 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:00,874 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:00,875 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:00,875 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:00,876 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:01,211 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:01,246 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:01,247 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:01,247 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:01,248 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:01,798 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:01,799 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:01,799 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:01,800 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:02,212 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:02,238 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:02,239 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:02,239 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:02,240 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:02,683 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:02,683 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:02,683 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:02,684 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:03,127 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:03,128 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:03,128 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:03,129 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:03,213 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:03,593 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:03,593 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:03,593 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:03,594 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:04,036 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:04,037 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:04,037 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:04,038 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:04,213 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:04,457 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:04,458 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:04,458 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:04,458 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:03:04,459 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:04,859 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:04,860 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:04,860 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:04,861 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:04,880 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 09:03:04,880 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: stop_status -2023-10-23 09:03:05,214 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:05,357 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:05,358 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:05,358 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:05,359 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:05,793 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:05,794 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:05,794 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:05,795 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:06,215 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:06,238 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:06,239 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:06,239 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:06,240 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:06,691 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:06,692 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:06,692 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:06,693 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:07,153 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:07,153 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:07,154 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:07,155 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:07,215 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:07,216 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/output.log -2023-10-23 09:03:07,629 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:07,629 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:07,629 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:07,630 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:08,020 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:08,021 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:08,021 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:08,022 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:08,216 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:08,446 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:08,447 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:08,447 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:08,449 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:08,947 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:08,948 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:08,948 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:08,949 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:09,217 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:09,423 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:09,424 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:09,424 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:09,425 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:09,808 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:09,809 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:09,809 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:09,809 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:03:09,810 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:10,217 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:10,328 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:10,329 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:10,329 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:10,329 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:10,808 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:10,808 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:10,809 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:10,809 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:11,218 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:11,274 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:11,274 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:11,274 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:11,275 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:11,727 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:11,728 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:11,728 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:11,728 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:12,121 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:12,121 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:12,121 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:12,122 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:12,219 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:12,558 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:12,558 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:12,559 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:12,559 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:13,030 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:13,030 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:13,031 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:13,031 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:13,219 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:13,457 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:13,457 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:13,458 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:13,458 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:13,896 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:13,897 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:13,897 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:13,898 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:14,220 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:14,270 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:14,271 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:14,271 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:14,271 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:14,741 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:14,741 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:14,741 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:14,742 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:15,221 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:15,276 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:15,276 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:15,276 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:15,276 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:03:15,277 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:15,654 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:15,655 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:15,655 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:15,655 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:16,070 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:16,070 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:16,071 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:16,071 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:16,222 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:16,467 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:16,468 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:16,468 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:16,469 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:16,974 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:16,975 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:16,975 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:16,976 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:17,222 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:17,498 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:17,498 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:17,499 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:17,499 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:17,944 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:17,944 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:17,944 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:17,945 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:18,223 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:18,395 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:18,395 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:18,395 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:18,396 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:18,829 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:18,829 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:18,829 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:18,831 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:19,217 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:19,217 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:19,217 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:19,218 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:19,224 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:19,645 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:19,646 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:19,646 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:19,647 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:19,880 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 09:03:19,881 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: stop_status -2023-10-23 09:03:20,190 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:20,190 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:20,190 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:20,191 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:20,224 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:20,703 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:20,703 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:20,704 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:20,704 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:03:20,704 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:21,224 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:21,224 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:21,224 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:21,226 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:21,226 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:21,781 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:21,782 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:21,782 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:21,783 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:22,226 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:22,297 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:22,298 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:22,298 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:22,299 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:22,746 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:22,746 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:22,746 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:22,747 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:23,195 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:23,196 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:23,196 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:23,197 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:23,227 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:23,651 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:23,651 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:23,652 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:23,652 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:24,027 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:24,027 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:24,027 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:24,028 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:24,228 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:24,497 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:24,498 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:24,498 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:24,499 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:24,899 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:24,899 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:24,899 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:24,900 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:25,229 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:25,324 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:25,324 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:25,324 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:25,325 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:25,914 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:25,914 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:25,915 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:25,915 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:03:25,916 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:26,229 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:26,394 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:26,394 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:26,395 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:26,395 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:26,839 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:26,839 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:26,839 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:26,840 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:27,230 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:27,269 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:27,270 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:27,270 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:27,271 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:27,672 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:27,673 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:27,673 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:27,674 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:28,153 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:28,154 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:28,154 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:28,155 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:28,231 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:28,584 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:28,585 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:28,585 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:28,586 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:29,064 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:29,064 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:29,064 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:29,066 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:29,231 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:29,231 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/output.log -2023-10-23 09:03:29,452 DEBUG SenderThread:51707 [sender.py:send():375] send: stats -2023-10-23 09:03:29,506 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:29,507 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:29,507 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:29,508 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:29,896 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:29,896 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:29,897 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:29,897 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:30,232 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:30,320 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:30,321 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:30,321 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:30,322 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:30,738 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:30,738 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:30,739 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:30,739 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:31,130 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:31,130 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:31,131 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:31,131 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:03:31,131 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:31,232 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:31,517 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:31,517 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:31,518 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:31,518 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:31,947 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:31,947 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:31,948 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:31,948 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:32,233 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:32,529 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:32,530 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:32,530 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:32,531 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:33,006 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:33,007 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:33,007 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:33,007 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:33,234 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:33,458 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:33,459 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:33,459 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:33,460 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:33,894 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:33,894 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:33,895 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:33,895 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:34,234 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:34,341 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:34,341 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:34,341 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:34,342 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:34,800 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:34,801 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:34,801 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:34,802 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:34,888 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 09:03:34,888 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: stop_status -2023-10-23 09:03:35,235 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:35,246 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:35,246 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:35,246 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:35,247 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:35,687 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:35,688 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:35,688 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:35,688 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:36,106 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:36,107 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:36,107 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:36,112 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:36,236 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:36,633 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:36,634 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:36,634 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:36,634 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:03:36,635 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:37,071 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:37,072 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:37,072 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:37,073 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:37,237 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:37,514 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:37,514 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:37,515 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:37,518 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:37,912 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:37,913 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:37,913 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:37,914 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:38,237 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:38,343 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:38,344 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:38,344 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:38,345 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:38,866 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:38,866 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:38,867 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:38,867 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:39,238 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:39,296 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:39,296 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:39,296 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:39,298 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:39,820 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:39,821 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:39,821 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:39,822 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:40,239 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:40,260 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:40,261 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:40,261 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:40,262 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:40,682 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:40,682 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:40,682 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:40,683 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:41,130 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:41,131 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:41,131 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:41,132 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:41,239 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:41,600 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:41,600 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:41,601 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:41,601 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:42,162 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:42,162 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:42,163 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:42,163 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:03:42,163 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:42,240 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:42,559 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:42,560 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:42,560 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:42,561 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:43,148 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:03:43,148 DEBUG SenderThread:51707 [sender.py:send():375] send: history -2023-10-23 09:03:43,148 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:03:43,149 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:43,241 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:43,490 DEBUG SenderThread:51707 [sender.py:send():375] send: exit -2023-10-23 09:03:43,490 INFO SenderThread:51707 [sender.py:send_exit():598] handling exit code: 255 -2023-10-23 09:03:43,491 INFO SenderThread:51707 [sender.py:send_exit():600] handling runtime: 164 -2023-10-23 09:03:43,492 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:43,492 INFO SenderThread:51707 [sender.py:send_exit():606] send defer -2023-10-23 09:03:43,492 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:03:43,492 INFO HandlerThread:51707 [handler.py:handle_request_defer():170] handle defer: 0 -2023-10-23 09:03:43,492 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: defer -2023-10-23 09:03:43,492 INFO SenderThread:51707 [sender.py:send_request_defer():622] handle sender defer: 0 -2023-10-23 09:03:43,492 INFO SenderThread:51707 [sender.py:transition_state():626] send defer: 1 -2023-10-23 09:03:43,492 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:03:43,492 INFO HandlerThread:51707 [handler.py:handle_request_defer():170] handle defer: 1 -2023-10-23 09:03:43,492 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: defer -2023-10-23 09:03:43,492 INFO SenderThread:51707 [sender.py:send_request_defer():622] handle sender defer: 1 -2023-10-23 09:03:43,492 INFO SenderThread:51707 [sender.py:transition_state():626] send defer: 2 -2023-10-23 09:03:43,493 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:03:43,493 INFO HandlerThread:51707 [handler.py:handle_request_defer():170] handle defer: 2 -2023-10-23 09:03:43,493 INFO HandlerThread:51707 [system_monitor.py:finish():190] Stopping system monitor -2023-10-23 09:03:43,493 DEBUG SystemMonitor:51707 [system_monitor.py:_start():166] Finished system metrics aggregation loop -2023-10-23 09:03:43,493 INFO HandlerThread:51707 [interfaces.py:finish():202] Joined cpu monitor -2023-10-23 09:03:43,493 DEBUG SystemMonitor:51707 [system_monitor.py:_start():170] Publishing last batch of metrics -2023-10-23 09:03:43,493 INFO HandlerThread:51707 [interfaces.py:finish():202] Joined disk monitor -2023-10-23 09:03:43,598 INFO HandlerThread:51707 [interfaces.py:finish():202] Joined gpu monitor -2023-10-23 09:03:43,598 INFO HandlerThread:51707 [interfaces.py:finish():202] Joined memory monitor -2023-10-23 09:03:43,599 INFO HandlerThread:51707 [interfaces.py:finish():202] Joined network monitor -2023-10-23 09:03:43,599 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: defer -2023-10-23 09:03:43,599 INFO SenderThread:51707 [sender.py:send_request_defer():622] handle sender defer: 2 -2023-10-23 09:03:43,599 INFO SenderThread:51707 [sender.py:transition_state():626] send defer: 3 -2023-10-23 09:03:43,599 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:03:43,599 INFO HandlerThread:51707 [handler.py:handle_request_defer():170] handle defer: 3 -2023-10-23 09:03:43,599 DEBUG SenderThread:51707 [sender.py:send():375] send: stats -2023-10-23 09:03:43,600 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: defer -2023-10-23 09:03:43,600 INFO SenderThread:51707 [sender.py:send_request_defer():622] handle sender defer: 3 -2023-10-23 09:03:43,600 INFO SenderThread:51707 [sender.py:transition_state():626] send defer: 4 -2023-10-23 09:03:43,600 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:03:43,600 INFO HandlerThread:51707 [handler.py:handle_request_defer():170] handle defer: 4 -2023-10-23 09:03:43,600 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: defer -2023-10-23 09:03:43,600 INFO SenderThread:51707 [sender.py:send_request_defer():622] handle sender defer: 4 -2023-10-23 09:03:43,600 INFO SenderThread:51707 [sender.py:transition_state():626] send defer: 5 -2023-10-23 09:03:43,600 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:03:43,600 INFO HandlerThread:51707 [handler.py:handle_request_defer():170] handle defer: 5 -2023-10-23 09:03:43,600 DEBUG SenderThread:51707 [sender.py:send():375] send: summary -2023-10-23 09:03:43,601 INFO SenderThread:51707 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:03:43,601 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: defer -2023-10-23 09:03:43,601 INFO SenderThread:51707 [sender.py:send_request_defer():622] handle sender defer: 5 -2023-10-23 09:03:43,601 INFO SenderThread:51707 [sender.py:transition_state():626] send defer: 6 -2023-10-23 09:03:43,601 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:03:43,601 INFO HandlerThread:51707 [handler.py:handle_request_defer():170] handle defer: 6 -2023-10-23 09:03:43,601 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: defer -2023-10-23 09:03:43,601 INFO SenderThread:51707 [sender.py:send_request_defer():622] handle sender defer: 6 -2023-10-23 09:03:43,602 INFO SenderThread:51707 [sender.py:transition_state():626] send defer: 7 -2023-10-23 09:03:43,602 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:03:43,602 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:03:43,602 INFO HandlerThread:51707 [handler.py:handle_request_defer():170] handle defer: 7 -2023-10-23 09:03:43,602 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: defer -2023-10-23 09:03:43,602 INFO SenderThread:51707 [sender.py:send_request_defer():622] handle sender defer: 7 -2023-10-23 09:03:44,241 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:44,490 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: poll_exit -2023-10-23 09:03:46,243 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/output.log -2023-10-23 09:03:47,200 INFO SenderThread:51707 [sender.py:transition_state():626] send defer: 8 -2023-10-23 09:03:47,200 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: poll_exit -2023-10-23 09:03:47,200 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:03:47,200 INFO HandlerThread:51707 [handler.py:handle_request_defer():170] handle defer: 8 -2023-10-23 09:03:47,200 DEBUG SenderThread:51707 [sender.py:send_request():402] send_request: defer -2023-10-23 09:03:47,200 INFO SenderThread:51707 [sender.py:send_request_defer():622] handle sender defer: 8 -2023-10-23 09:03:47,210 INFO SenderThread:51707 [sender.py:transition_state():626] send defer: 9 -2023-10-23 09:03:47,210 DEBUG HandlerThread:51707 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:03:47,210 DEBUG SenderThread:51707 [sender.py:send():375] send: artifact -2023-10-23 09:03:47,210 INFO HandlerThread:51707 [handler.py:handle_request_defer():170] handle defer: 9 -2023-10-23 09:03:47,243 INFO Thread-12 :51707 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/output.log -2023-10-23 09:03:47,294 WARNING StreamThr :51707 [internal.py:is_dead():414] Internal process exiting, parent pid 51105 disappeared -2023-10-23 09:03:47,295 ERROR StreamThr :51707 [internal.py:wandb_internal():152] Internal process shutdown. -2023-10-23 09:03:48,212 INFO HandlerThread:51707 [handler.py:finish():845] shutting down handler -2023-10-23 09:03:48,212 INFO WriterThread:51707 [datastore.py:close():298] close: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/run-141dzr9c.wandb -2023-10-23 09:03:48,216 INFO SenderThread:51707 [sender.py:send_artifact():1474] sent artifact job-https___github.com_junchen14_MiniGPT-4_finetune_train.py - {'id': 'QXJ0aWZhY3Q6NjEzMjczODU5', 'digest': 'e1357229d7190882bf8b2dba86a37a56', 'state': 'COMMITTED', 'aliases': [{'artifactCollectionName': 'job-https___github.com_junchen14_MiniGPT-4_finetune_train.py', 'alias': 'latest'}, {'artifactCollectionName': 'job-https___github.com_junchen14_MiniGPT-4_finetune_train.py', 'alias': 'v0'}], 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjEwOTExOTI4MQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6NjEzMjczODU5', 'versionIndex': 0}}, 'version': 'v0'} -2023-10-23 09:03:48,216 INFO SenderThread:51707 [sender.py:finish():1550] shutting down sender -2023-10-23 09:03:48,216 INFO SenderThread:51707 [dir_watcher.py:finish():365] shutting down directory watcher -2023-10-23 09:03:48,244 INFO SenderThread:51707 [dir_watcher.py:finish():395] scan: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files -2023-10-23 09:03:48,245 INFO SenderThread:51707 [dir_watcher.py:finish():409] scan save: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/conda-environment.yaml conda-environment.yaml -2023-10-23 09:03:48,245 INFO SenderThread:51707 [dir_watcher.py:finish():409] scan save: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/config.yaml config.yaml -2023-10-23 09:03:48,247 INFO SenderThread:51707 [dir_watcher.py:finish():409] scan save: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json wandb-summary.json -2023-10-23 09:03:48,250 INFO SenderThread:51707 [dir_watcher.py:finish():409] scan save: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/output.log output.log -2023-10-23 09:03:48,252 INFO SenderThread:51707 [dir_watcher.py:finish():409] scan save: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/requirements.txt requirements.txt -2023-10-23 09:03:48,252 INFO SenderThread:51707 [dir_watcher.py:finish():409] scan save: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-metadata.json wandb-metadata.json -2023-10-23 09:03:48,253 INFO SenderThread:51707 [file_pusher.py:finish():168] shutting down file pusher -2023-10-23 09:03:48,255 INFO SenderThread:51707 [file_pusher.py:join():173] waiting for file pusher -2023-10-23 09:03:48,833 INFO wandb-upload_0:51707 [upload_job.py:push():137] Uploaded file /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/conda-environment.yaml -2023-10-23 09:03:49,124 INFO wandb-upload_1:51707 [upload_job.py:push():137] Uploaded file /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/config.yaml -2023-10-23 09:03:49,247 INFO wandb-upload_2:51707 [upload_job.py:push():137] Uploaded file /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/wandb-summary.json -2023-10-23 09:03:49,256 INFO wandb-upload_4:51707 [upload_job.py:push():137] Uploaded file /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/requirements.txt -2023-10-23 09:03:49,326 INFO wandb-upload_3:51707 [upload_job.py:push():137] Uploaded file /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/files/output.log diff --git a/wandb/run-20231023_090058-141dzr9c/logs/debug.log b/wandb/run-20231023_090058-141dzr9c/logs/debug.log deleted file mode 100644 index d453118..0000000 --- a/wandb/run-20231023_090058-141dzr9c/logs/debug.log +++ /dev/null @@ -1,26 +0,0 @@ -2023-10-23 09:00:58,119 INFO MainThread:51105 [wandb_setup.py:_flush():76] Configure stats pid to 51105 -2023-10-23 09:00:58,119 INFO MainThread:51105 [wandb_setup.py:_flush():76] Loading settings from /home/chenj0g/.config/wandb/settings -2023-10-23 09:00:58,119 INFO MainThread:51105 [wandb_setup.py:_flush():76] Loading settings from /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/settings -2023-10-23 09:00:58,119 INFO MainThread:51105 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} -2023-10-23 09:00:58,119 INFO MainThread:51105 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'train.py', 'program': '/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/train.py'} -2023-10-23 09:00:58,119 INFO MainThread:51105 [wandb_init.py:_log_setup():507] Logging user logs to /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/logs/debug.log -2023-10-23 09:00:58,119 INFO MainThread:51105 [wandb_init.py:_log_setup():508] Logging internal logs to /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_090058-141dzr9c/logs/debug-internal.log -2023-10-23 09:00:58,119 INFO MainThread:51105 [wandb_init.py:init():547] calling init triggers -2023-10-23 09:00:58,119 INFO MainThread:51105 [wandb_init.py:init():554] wandb.init called with sweep_config: {} -config: {} -2023-10-23 09:00:58,119 INFO MainThread:51105 [wandb_init.py:init():595] starting backend -2023-10-23 09:00:58,119 INFO MainThread:51105 [wandb_init.py:init():599] setting up manager -2023-10-23 09:00:58,121 INFO MainThread:51105 [backend.py:_multiprocessing_setup():106] multiprocessing start_methods=fork,spawn,forkserver, using: spawn -2023-10-23 09:00:58,125 INFO MainThread:51105 [wandb_init.py:init():605] backend started and connected -2023-10-23 09:00:58,127 INFO MainThread:51105 [wandb_init.py:init():695] updated telemetry -2023-10-23 09:00:58,534 INFO MainThread:51105 [wandb_init.py:init():732] communicating run to backend with 60.0 second timeout -2023-10-23 09:00:59,118 INFO MainThread:51105 [wandb_run.py:_on_init():2176] communicating current version -2023-10-23 09:00:59,434 INFO MainThread:51105 [wandb_run.py:_on_init():2185] got version response upgrade_message: "wandb version 0.15.12 is available! To upgrade, please run:\n $ pip install wandb --upgrade" - -2023-10-23 09:00:59,434 INFO MainThread:51105 [wandb_init.py:init():782] starting run threads in backend -2023-10-23 09:01:04,878 INFO MainThread:51105 [wandb_run.py:_console_start():2157] atexit reg -2023-10-23 09:01:04,878 INFO MainThread:51105 [wandb_run.py:_redirect():2012] redirect: SettingsConsole.WRAP_RAW -2023-10-23 09:01:04,878 INFO MainThread:51105 [wandb_run.py:_redirect():2077] Wrapping output streams. -2023-10-23 09:01:04,878 INFO MainThread:51105 [wandb_run.py:_redirect():2102] Redirects installed. -2023-10-23 09:01:04,879 INFO MainThread:51105 [wandb_init.py:init():824] run started, returning control to user process -2023-10-23 09:01:04,879 INFO MainThread:51105 [wandb_watch.py:watch():51] Watching diff --git a/wandb/run-20231023_090058-141dzr9c/run-141dzr9c.wandb b/wandb/run-20231023_090058-141dzr9c/run-141dzr9c.wandb deleted file mode 100644 index 4459c86..0000000 Binary files a/wandb/run-20231023_090058-141dzr9c/run-141dzr9c.wandb and /dev/null differ diff --git a/wandb/run-20231023_091029-txn4i28h/files/conda-environment.yaml b/wandb/run-20231023_091029-txn4i28h/files/conda-environment.yaml deleted file mode 100644 index cfced9e..0000000 --- a/wandb/run-20231023_091029-txn4i28h/files/conda-environment.yaml +++ /dev/null @@ -1,297 +0,0 @@ -name: eye -channels: - - pytorch - - defaults -dependencies: - - _libgcc_mutex=0.1=main - - _openmp_mutex=5.1=1_gnu - - blas=1.0=mkl - - brotlipy=0.7.0=py39h27cfd23_1003 - - bzip2=1.0.8=h7b6447c_0 - - ca-certificates=2023.01.10=h06a4308_0 - - certifi=2022.12.7=py39h06a4308_0 - - cffi=1.15.1=py39h5eee18b_3 - - charset-normalizer=2.0.4=pyhd3eb1b0_0 - - cryptography=39.0.1=py39h9ce1e76_0 - - cudatoolkit=11.3.1=h2bc3f7f_2 - - ffmpeg=4.3=hf484d3e_0 - - flit-core=3.8.0=py39h06a4308_0 - - freetype=2.12.1=h4a9f257_0 - - giflib=5.2.1=h5eee18b_3 - - gmp=6.2.1=h295c915_3 - - gnutls=3.6.15=he1e5248_0 - - idna=3.4=py39h06a4308_0 - - intel-openmp=2021.4.0=h06a4308_3561 - - jpeg=9e=h5eee18b_1 - - lame=3.100=h7b6447c_0 - - lcms2=2.12=h3be6417_0 - - ld_impl_linux-64=2.38=h1181459_1 - - lerc=3.0=h295c915_0 - - libdeflate=1.17=h5eee18b_0 - - libffi=3.4.2=h6a678d5_6 - - libgcc-ng=11.2.0=h1234567_1 - - libgomp=11.2.0=h1234567_1 - - libiconv=1.16=h7f8727e_2 - - libidn2=2.3.2=h7f8727e_0 - - libpng=1.6.39=h5eee18b_0 - - libstdcxx-ng=11.2.0=h1234567_1 - - libtasn1=4.19.0=h5eee18b_0 - - libtiff=4.5.0=h6a678d5_2 - - libunistring=0.9.10=h27cfd23_0 - - libwebp=1.2.4=h11a3e52_1 - - libwebp-base=1.2.4=h5eee18b_1 - - lz4-c=1.9.4=h6a678d5_0 - - mkl=2021.4.0=h06a4308_640 - - mkl-service=2.4.0=py39h7f8727e_0 - - mkl_fft=1.3.1=py39hd3c417c_0 - - mkl_random=1.2.2=py39h51133e4_0 - - ncurses=6.4=h6a678d5_0 - - nettle=3.7.3=hbbd107a_1 - - numpy=1.23.5=py39h14f4228_0 - - numpy-base=1.23.5=py39h31eccc5_0 - - openh264=2.1.1=h4ff587b_0 - - openssl=1.1.1t=h7f8727e_0 - - pillow=9.4.0=py39h6a678d5_0 - - pip=23.0.1=py39h06a4308_0 - - pycparser=2.21=pyhd3eb1b0_0 - - pyopenssl=23.0.0=py39h06a4308_0 - - pysocks=1.7.1=py39h06a4308_0 - - python=3.9.16=h7a1cb2a_2 - - pytorch-mutex=1.0=cuda - - readline=8.2=h5eee18b_0 - - requests=2.28.1=py39h06a4308_1 - - setuptools=65.6.3=py39h06a4308_0 - - six=1.16.0=pyhd3eb1b0_1 - - sqlite=3.41.1=h5eee18b_0 - - tk=8.6.12=h1ccaba5_0 - - torchaudio=0.12.1=py39_cu113 - - torchvision=0.13.1=py39_cu113 - - typing_extensions=4.4.0=py39h06a4308_0 - - urllib3=1.26.15=py39h06a4308_0 - - wheel=0.38.4=py39h06a4308_0 - - xz=5.2.10=h5eee18b_1 - - zlib=1.2.13=h5eee18b_0 - - zstd=1.5.4=hc292b87_0 - - pip: - - accelerate==0.20.3 - - aiofiles==23.1.0 - - aiohttp==3.8.4 - - aiosignal==1.3.1 - - altair==4.2.2 - - antlr4-python3-runtime==4.9.3 - - anyio==3.6.2 - - appdirs==1.4.4 - - argon2-cffi==21.3.0 - - argon2-cffi-bindings==21.2.0 - - arrow==1.2.3 - - asttokens==2.2.1 - - async-timeout==4.0.2 - - attrs==22.2.0 - - backcall==0.2.0 - - beautifulsoup4==4.12.2 - - bitsandbytes==0.37.0 - - bleach==6.0.0 - - blis==0.7.9 - - braceexpand==0.1.7 - - catalogue==2.0.8 - - cchardet==2.1.7 - - chardet==5.1.0 - - click==8.1.3 - - cloudpickle==2.2.1 - - cmake==3.26.1 - - comm==0.1.3 - - confection==0.0.4 - - contourpy==1.0.7 - - cycler==0.11.0 - - cymem==2.0.7 - - datasets==2.13.1 - - debugpy==1.6.7 - - decorator==5.1.1 - - decord==0.6.0 - - defusedxml==0.7.1 - - dill==0.3.6 - - docker-pycreds==0.4.0 - - einops==0.6.0 - - entrypoints==0.4 - - executing==1.2.0 - - fairscale==0.4.13 - - fastapi==0.95.0 - - fastjsonschema==2.16.3 - - ffmpy==0.3.0 - - filelock==3.9.0 - - fire==0.5.0 - - fonttools==4.38.0 - - fqdn==1.5.1 - - frozenlist==1.3.3 - - fsspec==2023.4.0 - - gitdb==4.0.10 - - gitpython==3.1.31 - - gradio==3.24.1 - - gradio-client==0.0.8 - - h11==0.14.0 - - httpcore==0.16.3 - - httpx==0.23.3 - - huggingface-hub==0.16.4 - - imageio==2.31.1 - - importlib-metadata==6.2.0 - - importlib-resources==5.12.0 - - iopath==0.1.10 - - ipykernel==6.22.0 - - ipython==8.12.0 - - ipython-genutils==0.2.0 - - isoduration==20.11.0 - - jedi==0.18.2 - - jinja2==3.1.2 - - joblib==1.2.0 - - jsonpointer==2.3 - - jsonschema==4.17.3 - - jupyter-client==8.1.0 - - jupyter-core==5.3.0 - - jupyter-events==0.6.3 - - jupyter-server==2.5.0 - - jupyter-server-terminals==0.4.4 - - jupyterlab-pygments==0.2.2 - - kiwisolver==1.4.4 - - langcodes==3.3.0 - - lazy-loader==0.2 - - linkify-it-py==2.0.0 - - lit==16.0.0 - - llvmlite==0.39.1 - - loralib==0.1.1 - - markdown-it-py==2.2.0 - - markupsafe==2.1.2 - - matplotlib==3.7.0 - - matplotlib-inline==0.1.6 - - mdit-py-plugins==0.3.3 - - mdurl==0.1.2 - - mistune==2.0.5 - - mpmath==1.3.0 - - multidict==6.0.4 - - multiprocess==0.70.14 - - murmurhash==1.0.9 - - nbclassic==0.5.5 - - nbclient==0.7.3 - - nbconvert==7.3.0 - - nbformat==5.8.0 - - nest-asyncio==1.5.6 - - networkx==3.1 - - nltk==3.8.1 - - notebook==6.5.4 - - notebook-shim==0.2.2 - - numba==0.56.4 - - nvidia-cublas-cu11==11.10.3.66 - - nvidia-cuda-cupti-cu11==11.7.101 - - nvidia-cuda-nvrtc-cu11==11.7.99 - - nvidia-cuda-runtime-cu11==11.7.99 - - nvidia-cudnn-cu11==8.5.0.96 - - nvidia-cufft-cu11==10.9.0.58 - - nvidia-curand-cu11==10.2.10.91 - - nvidia-cusolver-cu11==11.4.0.1 - - nvidia-cusparse-cu11==11.7.4.91 - - nvidia-nccl-cu11==2.14.3 - - nvidia-nvtx-cu11==11.7.91 - - omegaconf==2.3.0 - - openai==0.27.0 - - opencv-python==4.7.0.72 - - orjson==3.8.9 - - packaging==23.0 - - pandas==2.0.0 - - pandocfilters==1.5.0 - - parso==0.8.3 - - pathlib==1.0.1 - - pathtools==0.1.2 - - pathy==0.10.1 - - peft==0.2.0 - - pexpect==4.8.0 - - pickleshare==0.7.5 - - platformdirs==3.2.0 - - policies==0.4.2 - - portalocker==2.7.0 - - preshed==3.0.8 - - progressbar2==4.2.0 - - prometheus-client==0.16.0 - - prompt-toolkit==3.0.38 - - protobuf==4.22.1 - - psutil==5.9.4 - - ptyprocess==0.7.0 - - pure-eval==0.2.2 - - pyarrow==12.0.1 - - pycocoevalcap==1.2 - - pycocotools==2.0.6 - - pycryptodome==3.17 - - pydantic==1.10.7 - - pydub==0.25.1 - - pygments==2.14.0 - - pynndescent==0.5.8 - - pyparsing==3.0.9 - - pyrsistent==0.19.3 - - python-dateutil==2.8.2 - - python-json-logger==2.0.7 - - python-multipart==0.0.6 - - python-utils==3.7.0 - - pytz==2023.3 - - pywavelets==1.4.1 - - pyyaml==6.0 - - pyzmq==25.0.2 - - regex==2022.10.31 - - rfc3339-validator==0.1.4 - - rfc3986==1.5.0 - - rfc3986-validator==0.1.1 - - safetensors==0.3.1 - - scikit-image==0.21.0 - - scikit-learn==1.2.2 - - scipy==1.10.1 - - seaborn==0.12.2 - - semantic-version==2.10.0 - - send2trash==1.8.0 - - sentence-transformers==2.2.2 - - sentencepiece==0.1.97 - - sentry-sdk==1.19.1 - - setproctitle==1.3.2 - - smart-open==6.3.0 - - smmap==5.0.0 - - sniffio==1.3.0 - - soupsieve==2.4 - - spacy==3.5.1 - - spacy-legacy==3.0.12 - - spacy-loggers==1.0.4 - - srsly==2.4.6 - - stack-data==0.6.2 - - starlette==0.26.1 - - submitit==1.4.5 - - sympy==1.11.1 - - tenacity==8.2.2 - - termcolor==2.3.0 - - terminado==0.17.1 - - thinc==8.1.9 - - threadpoolctl==3.1.0 - - tifffile==2023.4.12 - - timm==0.6.13 - - tinycss2==1.2.1 - - tokenizers==0.13.3 - - toolz==0.12.0 - - torch==2.0.0 - - tornado==6.2 - - tqdm==4.64.1 - - traitlets==5.9.0 - - transformers==4.31.0 - - triton==2.0.0 - - typer==0.7.0 - - tzdata==2023.3 - - uc-micro-py==1.0.1 - - umap-learn==0.5.3 - - uri-template==1.2.0 - - uvicorn==0.21.1 - - visual-genome==1.1.1 - - wandb==0.14.2 - - wasabi==1.1.1 - - wcwidth==0.2.6 - - webcolors==1.13 - - webdataset==0.2.48 - - webencodings==0.5.1 - - websocket-client==1.5.1 - - websockets==11.0.1 - - xxhash==3.2.0 - - yarl==1.8.2 - - zipp==3.14.0 -prefix: /home/chenj0g/anaconda3/envs/eye diff --git a/wandb/run-20231023_091029-txn4i28h/files/config.yaml b/wandb/run-20231023_091029-txn4i28h/files/config.yaml deleted file mode 100644 index b1eeb43..0000000 --- a/wandb/run-20231023_091029-txn4i28h/files/config.yaml +++ /dev/null @@ -1,38 +0,0 @@ -wandb_version: 1 - -_wandb: - desc: null - value: - python_version: 3.9.16 - cli_version: 0.14.2 - framework: huggingface - huggingface_version: 4.31.0 - is_jupyter_run: false - is_kaggle_kernel: false - start_time: 1698041430.035569 - t: - 1: - - 1 - - 11 - - 41 - - 49 - - 55 - - 63 - - 71 - 2: - - 1 - - 11 - - 41 - - 49 - - 55 - - 63 - - 71 - 3: - - 1 - - 13 - - 23 - 4: 3.9.16 - 5: 0.14.2 - 6: 4.31.0 - 8: - - 5 diff --git a/wandb/run-20231023_091029-txn4i28h/files/output.log b/wandb/run-20231023_091029-txn4i28h/files/output.log deleted file mode 100644 index dccb2b0..0000000 --- a/wandb/run-20231023_091029-txn4i28h/files/output.log +++ /dev/null @@ -1,2 +0,0 @@ -/home/chenj0g/anaconda3/envs/eye/lib/python3.9/site-packages/torch/utils/checkpoint.py:31: UserWarning: None of the inputs have requires_grad=True. Gradients will be None - warnings.warn("None of the inputs have requires_grad=True. Gradients will be None") \ No newline at end of file diff --git a/wandb/run-20231023_091029-txn4i28h/files/requirements.txt b/wandb/run-20231023_091029-txn4i28h/files/requirements.txt deleted file mode 100644 index 75bc393..0000000 --- a/wandb/run-20231023_091029-txn4i28h/files/requirements.txt +++ /dev/null @@ -1,246 +0,0 @@ -accelerate==0.20.3 -aiofiles==23.2.1 -aiohttp==3.8.4 -aiosignal==1.3.1 -altair==5.1.2 -antlr4-python3-runtime==4.9.3 -anyio==3.6.2 -appdirs==1.4.4 -argon2-cffi-bindings==21.2.0 -argon2-cffi==21.3.0 -arrow==1.2.3 -asttokens==2.2.1 -async-timeout==4.0.2 -attrs==22.2.0 -backcall==0.2.0 -beautifulsoup4==4.12.2 -bitsandbytes==0.37.0 -bleach==6.0.0 -blis==0.7.9 -braceexpand==0.1.7 -brotlipy==0.7.0 -catalogue==2.0.8 -cchardet==2.1.7 -certifi==2022.12.7 -cffi==1.15.1 -chardet==5.1.0 -charset-normalizer==2.0.4 -click==8.1.3 -cloudpickle==2.2.1 -cmake==3.26.1 -comm==0.1.3 -confection==0.0.4 -contourpy==1.0.7 -cryptography==39.0.1 -cycler==0.11.0 -cymem==2.0.7 -datasets==2.13.1 -debugpy==1.6.7 -decorator==5.1.1 -decord==0.6.0 -defusedxml==0.7.1 -dill==0.3.6 -docker-pycreds==0.4.0 -einops==0.6.0 -entrypoints==0.4 -executing==1.2.0 -fairscale==0.4.13 -fastapi==0.95.0 -fastjsonschema==2.16.3 -ffmpy==0.3.1 -filelock==3.9.0 -fire==0.5.0 -flit-core==3.8.0 -fonttools==4.38.0 -fqdn==1.5.1 -frozenlist==1.3.3 -fsspec==2023.9.2 -gitdb==4.0.10 -gitpython==3.1.31 -gradio-client==0.6.0 -gradio==3.24.1 -h11==0.14.0 -httpcore==0.18.0 -httpx==0.25.0 -huggingface-hub==0.18.0 -idna==3.4 -imageio==2.31.1 -importlib-metadata==6.2.0 -importlib-resources==5.12.0 -iopath==0.1.10 -ipykernel==6.22.0 -ipython-genutils==0.2.0 -ipython==8.12.0 -isoduration==20.11.0 -jedi==0.18.2 -jinja2==3.1.2 -joblib==1.2.0 -jsonpointer==2.3 -jsonschema==4.17.3 -jupyter-client==8.1.0 -jupyter-core==5.3.0 -jupyter-events==0.6.3 -jupyter-server-terminals==0.4.4 -jupyter-server==2.5.0 -jupyterlab-pygments==0.2.2 -kiwisolver==1.4.4 -langcodes==3.3.0 -lazy-loader==0.2 -linkify-it-py==2.0.2 -lit==16.0.0 -llvmlite==0.39.1 -loralib==0.1.1 -markdown-it-py==2.2.0 -markupsafe==2.1.2 -matplotlib-inline==0.1.6 -matplotlib==3.7.0 -mdit-py-plugins==0.3.3 -mdurl==0.1.2 -mistune==2.0.5 -mkl-fft==1.3.1 -mkl-random==1.2.2 -mkl-service==2.4.0 -mpmath==1.3.0 -multidict==6.0.4 -multiprocess==0.70.14 -murmurhash==1.0.9 -nbclassic==0.5.5 -nbclient==0.7.3 -nbconvert==7.3.0 -nbformat==5.8.0 -nest-asyncio==1.5.6 -networkx==3.1 -nltk==3.8.1 -notebook-shim==0.2.2 -notebook==6.5.4 -numba==0.56.4 -numpy==1.23.5 -nvidia-cublas-cu11==11.10.3.66 -nvidia-cuda-cupti-cu11==11.7.101 -nvidia-cuda-nvrtc-cu11==11.7.99 -nvidia-cuda-runtime-cu11==11.7.99 -nvidia-cudnn-cu11==8.5.0.96 -nvidia-cufft-cu11==10.9.0.58 -nvidia-curand-cu11==10.2.10.91 -nvidia-cusolver-cu11==11.4.0.1 -nvidia-cusparse-cu11==11.7.4.91 -nvidia-nccl-cu11==2.14.3 -nvidia-nvtx-cu11==11.7.91 -omegaconf==2.3.0 -openai==0.27.0 -opencv-python==4.7.0.72 -orjson==3.9.8 -packaging==23.0 -pandas==2.0.0 -pandocfilters==1.5.0 -parso==0.8.3 -pathlib==1.0.1 -pathtools==0.1.2 -pathy==0.10.1 -peft==0.2.0 -pexpect==4.8.0 -pickleshare==0.7.5 -pillow==9.4.0 -pip==23.0.1 -platformdirs==3.2.0 -policies==0.4.2 -portalocker==2.7.0 -preshed==3.0.8 -progressbar2==4.2.0 -prometheus-client==0.16.0 -prompt-toolkit==3.0.38 -protobuf==4.22.1 -psutil==5.9.4 -ptyprocess==0.7.0 -pure-eval==0.2.2 -pyarrow==12.0.1 -pycocoevalcap==1.2 -pycocotools==2.0.6 -pycparser==2.21 -pycryptodome==3.17 -pydantic==1.10.7 -pydub==0.25.1 -pygments==2.14.0 -pynndescent==0.5.8 -pyopenssl==23.0.0 -pyparsing==3.0.9 -pyrsistent==0.19.3 -pysocks==1.7.1 -python-dateutil==2.8.2 -python-json-logger==2.0.7 -python-multipart==0.0.6 -python-utils==3.7.0 -pytz==2023.3 -pywavelets==1.4.1 -pyyaml==6.0 -pyzmq==25.0.2 -regex==2022.10.31 -requests==2.28.1 -rfc3339-validator==0.1.4 -rfc3986-validator==0.1.1 -rfc3986==1.5.0 -rpe-index==1.2.0 -safetensors==0.3.1 -scikit-image==0.21.0 -scikit-learn==1.2.2 -scipy==1.10.1 -seaborn==0.12.2 -semantic-version==2.10.0 -send2trash==1.8.0 -sentence-transformers==2.2.2 -sentencepiece==0.1.97 -sentry-sdk==1.19.1 -setproctitle==1.3.2 -setuptools==65.6.3 -six==1.16.0 -smart-open==6.3.0 -smmap==5.0.0 -sniffio==1.3.0 -soupsieve==2.4 -spacy-legacy==3.0.12 -spacy-loggers==1.0.4 -spacy==3.5.1 -srsly==2.4.6 -stack-data==0.6.2 -starlette==0.26.1 -submitit==1.4.5 -sympy==1.11.1 -tenacity==8.2.2 -termcolor==2.3.0 -terminado==0.17.1 -thinc==8.1.9 -threadpoolctl==3.1.0 -tifffile==2023.4.12 -timm==0.6.13 -tinycss2==1.2.1 -tokenizers==0.13.3 -toolz==0.12.0 -torch==2.0.0 -torchaudio==0.12.1 -torchvision==0.13.1 -tornado==6.2 -tqdm==4.64.1 -traitlets==5.9.0 -transformers==4.31.0 -triton==2.0.0 -typer==0.7.0 -typing-extensions==4.4.0 -tzdata==2023.3 -uc-micro-py==1.0.2 -umap-learn==0.5.3 -uri-template==1.2.0 -urllib3==1.26.15 -uvicorn==0.21.1 -visual-genome==1.1.1 -wandb==0.14.2 -wasabi==1.1.1 -wcwidth==0.2.6 -webcolors==1.13 -webdataset==0.2.48 -webencodings==0.5.1 -websocket-client==1.5.1 -websockets==11.0.1 -wheel==0.38.4 -xxhash==3.2.0 -yarl==1.8.2 -zipp==3.14.0 \ No newline at end of file diff --git a/wandb/run-20231023_091029-txn4i28h/files/wandb-metadata.json b/wandb/run-20231023_091029-txn4i28h/files/wandb-metadata.json deleted file mode 100644 index cfa596e..0000000 --- a/wandb/run-20231023_091029-txn4i28h/files/wandb-metadata.json +++ /dev/null @@ -1,374 +0,0 @@ -{ - "os": "Linux-3.10.0-1160.76.1.el7.x86_64-x86_64-with-glibc2.17", - "python": "3.9.16", - "heartbeatAt": "2023-10-23T06:10:31.384548", - "startedAt": "2023-10-23T06:10:29.992874", - "docker": null, - "cuda": null, - "args": [ - "--job_name", - "minigpt4_v2_test", - "--cfg-path", - "train_configs/minigpt_v2_finetune.yaml" - ], - "state": "running", - "program": "/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/train.py", - "codePath": "train.py", - "git": { - "remote": "https://github.com/junchen14/MiniGPT-4_finetune", - "commit": "5c3ec8bb730b2e9df23b4bb3eb24108f9c04a8d6" - }, - "email": "jun.chen@kaust.edu.sa", - "root": "/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune", - "host": "gpu201-16-l", - "username": "chenj0g", - "executable": "/home/chenj0g/anaconda3/envs/eye/bin/python", - "cpu_count": 64, - "cpu_count_logical": 64, - "cpu_freq": { - "current": 1996.2770000000016, - "min": 0.0, - "max": 0.0 - }, - "cpu_freq_per_core": [ - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - } - ], - "disk": { - "total": 251.85773849487305, - "used": 7.905696868896484 - }, - "gpu": "NVIDIA A100-SXM4-80GB", - "gpu_count": 2, - "gpu_devices": [ - { - "name": "NVIDIA A100-SXM4-80GB", - "memory_total": 85899345920 - }, - { - "name": "NVIDIA A100-SXM4-80GB", - "memory_total": 85899345920 - } - ], - "memory": { - "total": 503.7154769897461 - } -} diff --git a/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json b/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json deleted file mode 100644 index 012577d..0000000 --- a/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json +++ /dev/null @@ -1 +0,0 @@ -{"epoch": 0, "loss": 1.071101188659668, "_timestamp": 1698041570.1773224, "_runtime": 140.1417534351349, "_step": 258, "_wandb": {"runtime": 139}} \ No newline at end of file diff --git a/wandb/run-20231023_091029-txn4i28h/logs/debug-internal.log b/wandb/run-20231023_091029-txn4i28h/logs/debug-internal.log deleted file mode 100644 index 809670b..0000000 --- a/wandb/run-20231023_091029-txn4i28h/logs/debug-internal.log +++ /dev/null @@ -1,1375 +0,0 @@ -2023-10-23 09:10:30,034 INFO StreamThr :54597 [internal.py:wandb_internal():86] W&B internal server running at pid: 54597, started at: 2023-10-23 09:10:30.033566 -2023-10-23 09:10:30,034 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: status -2023-10-23 09:10:30,037 INFO WriterThread:54597 [datastore.py:open_for_write():85] open: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/run-txn4i28h.wandb -2023-10-23 09:10:30,038 DEBUG SenderThread:54597 [sender.py:send():375] send: header -2023-10-23 09:10:30,447 DEBUG SenderThread:54597 [sender.py:send():375] send: run -2023-10-23 09:10:31,051 INFO SenderThread:54597 [dir_watcher.py:__init__():219] watching files in: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files -2023-10-23 09:10:31,051 INFO SenderThread:54597 [sender.py:_start_run_threads():1124] run started: txn4i28h with start time 1698041430.035569 -2023-10-23 09:10:31,051 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:31,052 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:31,053 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: check_version -2023-10-23 09:10:31,053 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: check_version -2023-10-23 09:10:31,362 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: run_start -2023-10-23 09:10:31,367 DEBUG HandlerThread:54597 [system_info.py:__init__():31] System info init -2023-10-23 09:10:31,367 DEBUG HandlerThread:54597 [system_info.py:__init__():46] System info init done -2023-10-23 09:10:31,367 INFO HandlerThread:54597 [system_monitor.py:start():181] Starting system monitor -2023-10-23 09:10:31,367 INFO SystemMonitor:54597 [system_monitor.py:_start():145] Starting system asset monitoring threads -2023-10-23 09:10:31,368 INFO HandlerThread:54597 [system_monitor.py:probe():201] Collecting system info -2023-10-23 09:10:31,368 INFO SystemMonitor:54597 [interfaces.py:start():190] Started cpu monitoring -2023-10-23 09:10:31,368 INFO SystemMonitor:54597 [interfaces.py:start():190] Started disk monitoring -2023-10-23 09:10:31,368 INFO SystemMonitor:54597 [interfaces.py:start():190] Started gpu monitoring -2023-10-23 09:10:31,369 INFO SystemMonitor:54597 [interfaces.py:start():190] Started memory monitoring -2023-10-23 09:10:31,369 INFO SystemMonitor:54597 [interfaces.py:start():190] Started network monitoring -2023-10-23 09:10:31,384 DEBUG HandlerThread:54597 [system_info.py:probe():195] Probing system -2023-10-23 09:10:31,390 DEBUG HandlerThread:54597 [system_info.py:_probe_git():180] Probing git -2023-10-23 09:10:31,403 DEBUG HandlerThread:54597 [system_info.py:_probe_git():188] Probing git done -2023-10-23 09:10:31,403 DEBUG HandlerThread:54597 [system_info.py:probe():240] Probing system done -2023-10-23 09:10:31,403 DEBUG HandlerThread:54597 [system_monitor.py:probe():210] {'os': 'Linux-3.10.0-1160.76.1.el7.x86_64-x86_64-with-glibc2.17', 'python': '3.9.16', 'heartbeatAt': '2023-10-23T06:10:31.384548', 'startedAt': '2023-10-23T06:10:29.992874', 'docker': None, 'cuda': None, 'args': ('--job_name', 'minigpt4_v2_test', '--cfg-path', 'train_configs/minigpt_v2_finetune.yaml'), 'state': 'running', 'program': '/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/train.py', 'codePath': 'train.py', 'git': {'remote': 'https://github.com/junchen14/MiniGPT-4_finetune', 'commit': '5c3ec8bb730b2e9df23b4bb3eb24108f9c04a8d6'}, 'email': 'jun.chen@kaust.edu.sa', 'root': '/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune', 'host': 'gpu201-16-l', 'username': 'chenj0g', 'executable': '/home/chenj0g/anaconda3/envs/eye/bin/python', 'cpu_count': 64, 'cpu_count_logical': 64, 'cpu_freq': {'current': 1996.2770000000016, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}], 'disk': {'total': 251.85773849487305, 'used': 7.905696868896484}, 'gpu': 'NVIDIA A100-SXM4-80GB', 'gpu_count': 2, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}], 'memory': {'total': 503.7154769897461}} -2023-10-23 09:10:31,403 INFO HandlerThread:54597 [system_monitor.py:probe():211] Finished collecting system info -2023-10-23 09:10:31,403 INFO HandlerThread:54597 [system_monitor.py:probe():214] Publishing system info -2023-10-23 09:10:31,403 DEBUG HandlerThread:54597 [system_info.py:_save_pip():51] Saving list of pip packages installed into the current environment -2023-10-23 09:10:31,404 DEBUG HandlerThread:54597 [system_info.py:_save_pip():67] Saving pip packages done -2023-10-23 09:10:31,404 DEBUG HandlerThread:54597 [system_info.py:_save_conda():74] Saving list of conda packages installed into the current environment -2023-10-23 09:10:32,053 INFO Thread-12 :54597 [dir_watcher.py:_on_file_created():278] file/dir created: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:10:32,054 INFO Thread-12 :54597 [dir_watcher.py:_on_file_created():278] file/dir created: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/requirements.txt -2023-10-23 09:10:32,054 INFO Thread-12 :54597 [dir_watcher.py:_on_file_created():278] file/dir created: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/conda-environment.yaml -2023-10-23 09:10:37,055 DEBUG HandlerThread:54597 [system_info.py:_save_conda():86] Saving conda packages done -2023-10-23 09:10:37,057 INFO HandlerThread:54597 [system_monitor.py:probe():216] Finished publishing system info -2023-10-23 09:10:37,057 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/conda-environment.yaml -2023-10-23 09:10:37,058 INFO Thread-12 :54597 [dir_watcher.py:_on_file_created():278] file/dir created: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-metadata.json -2023-10-23 09:10:37,059 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:10:37,060 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: keepalive -2023-10-23 09:10:37,060 DEBUG SenderThread:54597 [sender.py:send():375] send: files -2023-10-23 09:10:37,060 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-metadata.json with policy now -2023-10-23 09:10:37,065 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 09:10:37,066 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: stop_status -2023-10-23 09:10:37,429 DEBUG SenderThread:54597 [sender.py:send():375] send: telemetry -2023-10-23 09:10:37,430 DEBUG SenderThread:54597 [sender.py:send():375] send: telemetry -2023-10-23 09:10:37,867 INFO wandb-upload_0:54597 [upload_job.py:push():137] Uploaded file /tmp/tmpi8sahgaawandb/npzi5ika-wandb-metadata.json -2023-10-23 09:10:40,430 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:10:42,060 INFO Thread-12 :54597 [dir_watcher.py:_on_file_created():278] file/dir created: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/output.log -2023-10-23 09:10:42,566 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:42,567 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:42,567 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:42,569 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:43,061 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:10:43,079 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:43,080 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:43,080 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:43,081 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:43,680 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:43,681 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:43,681 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:43,682 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:44,062 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/output.log -2023-10-23 09:10:44,062 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:10:44,148 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:44,148 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:44,149 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:44,149 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:44,651 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:44,652 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:44,652 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:44,654 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:45,063 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:10:45,120 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:45,121 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:45,121 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:45,122 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:45,631 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:45,632 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:45,632 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:45,632 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:10:45,633 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:46,064 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:10:46,151 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:46,152 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:46,152 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:46,153 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:46,700 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:46,701 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:46,701 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:46,703 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:47,064 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:10:47,182 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:47,183 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:47,183 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:47,184 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:47,656 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:47,657 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:47,657 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:47,658 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:48,065 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:10:48,206 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:48,207 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:48,207 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:48,207 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:48,676 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:48,676 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:48,677 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:48,677 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:49,066 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:10:49,129 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:49,130 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:49,130 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:49,132 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:49,579 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:49,580 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:49,580 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:49,581 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:50,067 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:10:50,137 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:50,138 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:50,138 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:50,139 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:50,686 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:50,687 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:50,687 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:50,687 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:10:50,688 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:51,069 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:10:51,112 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:51,112 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:51,112 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:51,113 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:51,581 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:51,581 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:51,581 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:51,583 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:52,069 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:10:52,088 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 09:10:52,089 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: stop_status -2023-10-23 09:10:52,093 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:52,329 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:52,330 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:52,330 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:52,532 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:52,532 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:52,532 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:52,533 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:53,007 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:53,007 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:53,007 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:53,008 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:53,070 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:10:53,555 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:53,556 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:53,556 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:53,557 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:54,001 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:54,001 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:54,001 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:54,003 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:54,070 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:10:54,510 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:54,511 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:54,511 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:54,512 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:55,015 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:55,015 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:55,016 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:55,016 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:55,071 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:10:55,457 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:55,458 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:55,458 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:55,459 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:55,937 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:55,938 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:55,938 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:55,938 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:10:55,939 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:56,072 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:10:56,406 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:56,407 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:56,407 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:56,409 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:56,859 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:56,860 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:56,860 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:56,861 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:57,073 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:10:57,356 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:57,357 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:57,357 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:57,358 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:57,910 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:57,910 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:57,910 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:57,911 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:58,073 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:10:58,390 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:58,391 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:58,391 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:58,392 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:58,848 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:58,848 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:58,849 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:58,850 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:59,075 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:10:59,420 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:59,420 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:59,421 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:59,421 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:59,871 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:59,871 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:10:59,872 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:59,873 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:00,075 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:00,395 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:00,396 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:00,396 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:00,397 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:00,900 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:00,900 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:00,901 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:00,902 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:01,076 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:01,379 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:01,379 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:01,383 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:11:01,621 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:01,622 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:01,838 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:01,838 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:01,838 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:01,839 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:02,076 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:02,077 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/config.yaml -2023-10-23 09:11:02,317 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:02,318 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:02,318 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:02,319 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:02,825 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:02,826 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:02,826 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:02,827 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:03,077 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:03,270 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:03,271 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:03,271 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:03,273 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:03,742 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:03,742 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:03,743 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:03,743 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:04,078 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:04,285 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:04,286 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:04,286 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:04,287 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:04,767 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:04,767 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:04,767 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:04,768 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:05,079 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:05,227 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:05,228 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:05,228 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:05,229 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:05,692 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:05,693 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:05,693 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:05,695 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:06,079 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:06,114 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:06,114 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:06,115 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:06,115 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:06,732 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:06,733 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:06,733 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:06,733 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:11:06,734 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:07,080 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:07,088 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 09:11:07,088 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: stop_status -2023-10-23 09:11:07,282 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:07,372 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:07,372 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:07,373 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:07,775 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:07,776 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:07,776 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:07,778 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:08,081 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:08,257 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:08,257 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:08,258 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:08,258 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:08,886 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:08,887 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:08,887 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:08,888 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:09,082 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:09,348 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:09,348 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:09,349 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:09,349 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:09,850 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:09,851 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:09,851 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:09,853 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:10,082 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:10,319 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:10,320 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:10,320 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:10,321 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:10,818 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:10,819 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:10,819 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:10,820 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:11,083 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:11,292 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:11,293 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:11,293 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:11,294 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:11,762 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:11,762 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:11,763 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:11,763 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:11:11,763 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:12,084 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:12,244 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:12,244 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:12,244 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:12,246 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:12,727 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:12,728 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:12,728 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:12,729 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:13,085 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:13,204 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:13,205 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:13,205 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:13,209 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:13,762 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:13,763 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:13,763 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:13,764 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:14,085 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:14,249 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:14,250 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:14,250 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:14,252 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:14,757 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:14,758 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:14,758 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:14,759 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:15,086 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:15,207 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:15,208 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:15,208 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:15,208 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:15,653 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:15,654 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:15,654 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:15,655 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:16,087 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:16,163 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:16,164 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:16,164 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:16,165 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:16,632 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:16,632 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:16,633 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:16,634 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:17,088 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:17,105 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:17,105 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:17,106 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:17,106 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:11:17,106 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:17,650 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:17,651 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:17,651 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:17,651 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:18,090 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:18,137 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:18,138 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:18,138 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:18,139 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:18,650 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:18,650 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:18,651 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:18,653 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:19,092 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:19,127 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:19,128 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:19,128 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:19,129 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:19,578 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:19,579 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:19,579 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:19,580 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:20,093 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:20,176 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:20,177 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:20,177 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:20,177 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:20,608 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:20,608 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:20,608 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:20,612 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:21,093 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:21,169 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:21,169 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:21,170 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:21,171 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:21,748 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:21,748 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:21,749 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:21,749 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:22,088 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 09:11:22,088 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: stop_status -2023-10-23 09:11:22,094 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:22,298 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:22,356 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:22,357 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:11:22,357 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:22,358 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:22,769 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:22,770 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:22,770 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:22,771 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:23,095 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:23,289 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:23,289 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:23,289 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:23,291 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:23,846 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:23,846 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:23,847 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:23,848 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:24,096 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:24,310 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:24,311 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:24,311 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:24,312 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:24,832 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:24,833 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:24,833 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:24,834 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:25,096 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:25,284 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:25,285 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:25,285 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:25,286 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:25,790 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:25,790 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:25,790 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:25,792 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:26,097 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:26,279 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:26,280 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:26,280 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:26,281 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:26,747 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:26,748 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:26,748 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:26,748 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:27,098 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:27,260 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:27,261 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:27,261 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:27,262 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:27,733 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:27,733 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:27,734 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:27,734 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:11:27,734 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:28,099 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:28,180 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:28,181 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:28,181 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:28,183 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:28,641 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:28,642 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:28,642 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:28,643 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:29,028 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:29,029 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:29,029 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:29,030 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:29,099 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:29,495 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:29,495 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:29,496 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:29,496 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:30,045 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:30,046 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:30,046 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:30,047 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:30,100 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:30,508 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:30,509 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:30,509 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:30,510 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:30,990 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:30,991 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:30,991 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:30,991 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:31,101 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:31,369 DEBUG SystemMonitor:54597 [system_monitor.py:_start():159] Starting system metrics aggregation loop -2023-10-23 09:11:31,371 DEBUG SenderThread:54597 [sender.py:send():375] send: stats -2023-10-23 09:11:31,540 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:31,541 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:31,541 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:31,542 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:32,060 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:32,061 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:32,061 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:32,062 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:32,102 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:32,502 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:32,502 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:32,503 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:32,503 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:32,997 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:32,998 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:32,998 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:32,998 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:11:32,999 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:33,102 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:33,467 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:33,468 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:33,468 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:33,469 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:33,994 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:33,995 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:33,995 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:33,996 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:34,103 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:34,464 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:34,465 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:34,465 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:34,466 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:35,003 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:35,004 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:35,004 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:35,005 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:35,104 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:35,469 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:35,470 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:35,470 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:35,471 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:35,929 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:35,930 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:35,930 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:35,931 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:36,105 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:36,383 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:36,383 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:36,384 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:36,384 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:36,899 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:36,899 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:36,899 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:36,900 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:37,088 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 09:11:37,088 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: stop_status -2023-10-23 09:11:37,105 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:37,495 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:37,496 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:37,496 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:37,497 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:37,873 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:37,874 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:37,874 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:37,875 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:38,107 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:38,402 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:38,403 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:38,403 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:38,404 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:11:38,405 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:38,928 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:38,929 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:38,929 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:38,930 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:39,109 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:39,516 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:39,516 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:39,517 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:39,517 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:39,995 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:39,996 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:39,996 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:39,996 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:40,110 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:40,447 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:40,448 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:40,448 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:40,449 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:41,075 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:41,076 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:41,076 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:41,077 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:41,111 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:41,528 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:41,529 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:41,529 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:41,530 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:42,011 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:42,012 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:42,012 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:42,013 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:42,112 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:42,564 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:42,565 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:42,565 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:42,566 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:43,031 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:43,032 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:43,032 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:43,033 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:43,112 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:43,488 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:43,489 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:43,489 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:43,489 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:11:43,490 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:43,935 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:43,935 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:43,935 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:43,936 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:44,113 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:44,437 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:44,437 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:44,438 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:44,438 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:44,888 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:44,889 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:44,889 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:44,890 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:45,114 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:45,355 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:45,356 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:45,356 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:45,356 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:45,816 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:45,816 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:45,817 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:45,817 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:46,115 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:46,301 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:46,301 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:46,302 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:46,302 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:46,773 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:46,774 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:46,774 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:46,775 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:47,116 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:47,298 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:47,299 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:47,299 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:47,300 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:47,772 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:47,772 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:47,773 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:47,773 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:48,116 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:48,217 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:48,218 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:48,218 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:48,219 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:48,690 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:48,691 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:48,691 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:48,692 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:11:48,693 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:49,117 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:49,135 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:49,136 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:49,136 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:49,136 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:49,674 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:49,675 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:49,675 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:49,676 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:50,118 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:50,119 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:50,119 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:50,119 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:50,120 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:50,630 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:50,630 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:50,631 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:50,631 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:51,119 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:51,155 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:51,156 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:51,156 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:51,157 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:51,623 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:51,624 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:51,624 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:51,625 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:52,054 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:52,054 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:52,055 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:52,055 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:52,088 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 09:11:52,088 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: stop_status -2023-10-23 09:11:52,121 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:52,598 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:52,599 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:52,599 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:52,600 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:53,122 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:53,166 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:53,166 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:53,166 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:53,167 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:53,687 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:53,687 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:53,688 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:53,688 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:54,122 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:54,143 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:54,144 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:54,144 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:54,144 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:11:54,145 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:54,601 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:54,601 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:54,602 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:54,602 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:55,031 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:55,032 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:55,032 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:55,034 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:55,123 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:55,591 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:55,592 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:55,592 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:55,593 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:56,049 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:56,049 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:56,049 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:56,050 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:56,124 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:56,553 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:56,554 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:56,554 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:56,555 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:57,051 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:57,052 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:57,052 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:57,053 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:57,125 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:57,478 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:57,479 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:57,479 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:57,479 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:57,909 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:57,910 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:57,910 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:57,911 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:58,125 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:58,318 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:58,318 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:58,318 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:58,319 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:58,868 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:58,869 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:58,869 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:58,871 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:59,126 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:11:59,339 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:59,340 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:59,340 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:59,340 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:11:59,341 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:59,788 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:59,788 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:11:59,789 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:59,789 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:00,127 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:00,237 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:00,238 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:00,238 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:00,239 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:00,687 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:00,687 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:00,687 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:00,688 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:01,128 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:01,219 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:01,219 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:01,219 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:01,220 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:01,371 DEBUG SenderThread:54597 [sender.py:send():375] send: stats -2023-10-23 09:12:01,667 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:01,668 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:01,668 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:01,669 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:02,129 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:02,197 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:02,198 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:02,198 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:02,199 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:02,726 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:02,727 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:02,727 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:02,728 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:03,129 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:03,154 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:03,154 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:03,154 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:03,155 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:03,624 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:03,625 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:03,625 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:03,626 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:04,130 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:04,155 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:04,155 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:04,155 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:04,156 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:04,742 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:04,743 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:04,743 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:04,743 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:12:04,744 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:05,131 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:05,196 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:05,197 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:05,197 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:05,198 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:05,680 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:05,681 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:05,681 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:05,682 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:06,132 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:06,166 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:06,167 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:06,167 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:06,168 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:06,674 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:06,675 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:06,675 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:06,676 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:07,119 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 09:12:07,120 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: stop_status -2023-10-23 09:12:07,123 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:07,132 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:07,339 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:07,339 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:07,340 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:07,594 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:07,595 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:07,595 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:07,595 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:08,061 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:08,061 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:08,062 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:08,063 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:08,133 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:08,489 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:08,490 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:08,490 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:08,491 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:08,940 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:08,941 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:08,941 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:08,943 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:09,134 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:09,413 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:09,414 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:09,414 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:09,415 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:09,894 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:09,895 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:09,895 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:09,895 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:12:09,896 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:10,135 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:10,372 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:10,372 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:10,372 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:10,373 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:10,880 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:10,880 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:10,880 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:10,881 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:11,135 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:11,394 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:11,395 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:11,395 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:11,396 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:11,949 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:11,950 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:11,950 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:11,951 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:12,136 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:12,471 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:12,472 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:12,472 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:12,473 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:12,992 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:12,993 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:12,993 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:12,994 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:13,137 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:13,524 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:13,525 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:13,525 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:13,526 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:14,029 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:14,030 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:14,030 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:14,031 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:14,138 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:14,592 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:14,592 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:14,593 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:14,593 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:15,091 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:15,092 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:15,092 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:15,092 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:12:15,093 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:15,139 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:15,639 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:15,640 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:15,640 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:15,641 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:16,118 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:16,119 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:16,119 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:16,120 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:16,139 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:16,546 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:16,547 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:16,547 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:16,548 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:17,105 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:17,106 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:17,106 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:17,107 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:17,140 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:17,577 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:17,577 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:17,577 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:17,578 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:18,141 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:18,170 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:18,171 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:18,171 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:18,172 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:18,667 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:18,667 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:18,668 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:18,668 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:19,094 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:19,094 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:19,094 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:19,097 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:19,142 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:19,692 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:19,693 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:19,693 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:19,694 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:20,142 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:20,228 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:20,229 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:20,229 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:20,229 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:12:20,230 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:20,672 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:20,672 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:20,672 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:20,673 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:21,143 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:21,156 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:21,157 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:21,157 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:21,158 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:21,718 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:21,718 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:21,718 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:21,719 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:22,119 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 09:12:22,120 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: stop_status -2023-10-23 09:12:22,144 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:22,186 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:22,415 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:22,416 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:22,416 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:22,786 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:22,787 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:22,787 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:22,788 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:23,145 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:23,324 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:23,325 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:23,325 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:23,325 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:23,809 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:23,809 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:23,810 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:23,810 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:24,145 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:24,276 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:24,277 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:24,277 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:24,278 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:24,797 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:24,797 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:24,797 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:24,798 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:25,146 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:25,336 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:25,336 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:25,337 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:25,337 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:12:25,337 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:25,813 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:25,813 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:25,813 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:25,814 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:26,147 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:26,328 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:26,329 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:26,329 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:26,330 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:26,770 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:26,771 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:26,771 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:26,772 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:27,147 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:27,263 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:27,264 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:27,264 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:27,265 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:27,653 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:27,653 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:27,654 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:27,654 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:28,148 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:28,219 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:28,219 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:28,220 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:28,220 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:28,685 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:28,686 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:28,686 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:28,687 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:29,145 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:29,146 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:29,146 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:29,148 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:29,149 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:29,703 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:29,704 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:29,704 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:29,705 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:30,149 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:30,184 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:30,184 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:30,184 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:30,185 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:30,729 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:30,730 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:30,730 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:30,730 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:12:30,731 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:31,150 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:31,236 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:31,237 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:31,237 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:31,237 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:31,372 DEBUG SenderThread:54597 [sender.py:send():375] send: stats -2023-10-23 09:12:31,738 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:31,738 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:31,738 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:31,739 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:32,151 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:32,248 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:32,248 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:32,248 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:32,249 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:32,654 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:32,655 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:32,655 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:32,656 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:33,108 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:33,108 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:33,109 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:33,110 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:33,152 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:33,666 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:33,667 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:33,667 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:33,668 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:34,152 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:34,210 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:34,211 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:34,211 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:34,211 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:34,654 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:34,654 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:34,654 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:34,655 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:35,101 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:35,101 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:35,101 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:35,102 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:35,153 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:35,651 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:35,651 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:35,651 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:35,652 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:36,098 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:36,098 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:36,098 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:36,098 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:12:36,099 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:36,154 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:36,644 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:36,645 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:36,645 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:36,646 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:37,150 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 09:12:37,150 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: stop_status -2023-10-23 09:12:37,153 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:37,155 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:37,409 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:37,409 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:37,410 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:37,582 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:37,583 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:37,583 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:37,584 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:38,155 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:38,180 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:38,180 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:38,180 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:38,181 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:38,727 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:38,728 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:38,728 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:38,728 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:39,156 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:39,226 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:39,227 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:39,227 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:39,229 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:39,735 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:39,736 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:39,736 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:39,737 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:40,157 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:40,215 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:40,216 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:40,216 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:40,217 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:40,690 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:40,690 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:40,690 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:40,691 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:41,158 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:41,260 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:41,261 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:41,261 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:41,261 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:12:41,262 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:41,715 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:41,715 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:41,715 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:41,716 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:42,158 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:42,285 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:42,286 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:42,286 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:42,287 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:42,766 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:42,767 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:42,767 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:42,768 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:43,159 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:43,212 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:43,213 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:43,213 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:43,214 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:43,666 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:43,666 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:43,666 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:43,667 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:44,163 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:44,226 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:44,227 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:44,227 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:44,228 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:44,732 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:44,732 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:44,732 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:44,733 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:45,159 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:45,159 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:45,160 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:45,161 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:45,163 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:45,663 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:45,664 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:45,664 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:45,665 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:46,115 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:46,115 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:46,116 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:46,116 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:46,164 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:46,562 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:46,563 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:46,563 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:46,563 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:12:46,564 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:47,112 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:47,113 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:47,113 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:47,116 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:47,165 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:47,557 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:47,557 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:47,557 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:47,558 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:48,121 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:48,122 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:48,122 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:48,123 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:48,166 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:48,601 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:48,601 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:48,601 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:48,602 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:49,082 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:49,083 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:49,083 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:49,084 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:49,167 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:49,646 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:49,646 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:49,646 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:49,648 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:50,167 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:50,179 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:50,179 DEBUG SenderThread:54597 [sender.py:send():375] send: history -2023-10-23 09:12:50,180 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:50,180 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:50,597 DEBUG SenderThread:54597 [sender.py:send():375] send: exit -2023-10-23 09:12:50,597 INFO SenderThread:54597 [sender.py:send_exit():598] handling exit code: 255 -2023-10-23 09:12:50,597 INFO SenderThread:54597 [sender.py:send_exit():600] handling runtime: 139 -2023-10-23 09:12:50,598 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:50,598 INFO SenderThread:54597 [sender.py:send_exit():606] send defer -2023-10-23 09:12:50,598 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:12:50,598 INFO HandlerThread:54597 [handler.py:handle_request_defer():170] handle defer: 0 -2023-10-23 09:12:50,598 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: defer -2023-10-23 09:12:50,598 INFO SenderThread:54597 [sender.py:send_request_defer():622] handle sender defer: 0 -2023-10-23 09:12:50,598 INFO SenderThread:54597 [sender.py:transition_state():626] send defer: 1 -2023-10-23 09:12:50,598 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:12:50,599 INFO HandlerThread:54597 [handler.py:handle_request_defer():170] handle defer: 1 -2023-10-23 09:12:50,599 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: defer -2023-10-23 09:12:50,599 INFO SenderThread:54597 [sender.py:send_request_defer():622] handle sender defer: 1 -2023-10-23 09:12:50,599 INFO SenderThread:54597 [sender.py:transition_state():626] send defer: 2 -2023-10-23 09:12:50,599 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:12:50,599 INFO HandlerThread:54597 [handler.py:handle_request_defer():170] handle defer: 2 -2023-10-23 09:12:50,599 INFO HandlerThread:54597 [system_monitor.py:finish():190] Stopping system monitor -2023-10-23 09:12:50,599 DEBUG SystemMonitor:54597 [system_monitor.py:_start():166] Finished system metrics aggregation loop -2023-10-23 09:12:50,600 INFO HandlerThread:54597 [interfaces.py:finish():202] Joined cpu monitor -2023-10-23 09:12:50,600 DEBUG SystemMonitor:54597 [system_monitor.py:_start():170] Publishing last batch of metrics -2023-10-23 09:12:50,600 INFO HandlerThread:54597 [interfaces.py:finish():202] Joined disk monitor -2023-10-23 09:12:50,702 INFO HandlerThread:54597 [interfaces.py:finish():202] Joined gpu monitor -2023-10-23 09:12:50,703 INFO HandlerThread:54597 [interfaces.py:finish():202] Joined memory monitor -2023-10-23 09:12:50,703 INFO HandlerThread:54597 [interfaces.py:finish():202] Joined network monitor -2023-10-23 09:12:50,703 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: defer -2023-10-23 09:12:50,703 INFO SenderThread:54597 [sender.py:send_request_defer():622] handle sender defer: 2 -2023-10-23 09:12:50,703 INFO SenderThread:54597 [sender.py:transition_state():626] send defer: 3 -2023-10-23 09:12:50,703 DEBUG SenderThread:54597 [sender.py:send():375] send: stats -2023-10-23 09:12:50,703 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:12:50,704 INFO HandlerThread:54597 [handler.py:handle_request_defer():170] handle defer: 3 -2023-10-23 09:12:50,704 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: defer -2023-10-23 09:12:50,704 INFO SenderThread:54597 [sender.py:send_request_defer():622] handle sender defer: 3 -2023-10-23 09:12:50,704 INFO SenderThread:54597 [sender.py:transition_state():626] send defer: 4 -2023-10-23 09:12:50,704 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:12:50,704 INFO HandlerThread:54597 [handler.py:handle_request_defer():170] handle defer: 4 -2023-10-23 09:12:50,704 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: defer -2023-10-23 09:12:50,704 INFO SenderThread:54597 [sender.py:send_request_defer():622] handle sender defer: 4 -2023-10-23 09:12:50,705 INFO SenderThread:54597 [sender.py:transition_state():626] send defer: 5 -2023-10-23 09:12:50,705 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:12:50,705 INFO HandlerThread:54597 [handler.py:handle_request_defer():170] handle defer: 5 -2023-10-23 09:12:50,705 DEBUG SenderThread:54597 [sender.py:send():375] send: summary -2023-10-23 09:12:50,706 INFO SenderThread:54597 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:50,706 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: defer -2023-10-23 09:12:50,706 INFO SenderThread:54597 [sender.py:send_request_defer():622] handle sender defer: 5 -2023-10-23 09:12:50,706 INFO SenderThread:54597 [sender.py:transition_state():626] send defer: 6 -2023-10-23 09:12:50,706 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:12:50,706 INFO HandlerThread:54597 [handler.py:handle_request_defer():170] handle defer: 6 -2023-10-23 09:12:50,706 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: defer -2023-10-23 09:12:50,706 INFO SenderThread:54597 [sender.py:send_request_defer():622] handle sender defer: 6 -2023-10-23 09:12:50,706 INFO SenderThread:54597 [sender.py:transition_state():626] send defer: 7 -2023-10-23 09:12:50,706 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:12:50,706 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:12:50,706 INFO HandlerThread:54597 [handler.py:handle_request_defer():170] handle defer: 7 -2023-10-23 09:12:50,707 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: defer -2023-10-23 09:12:50,707 INFO SenderThread:54597 [sender.py:send_request_defer():622] handle sender defer: 7 -2023-10-23 09:12:51,168 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:51,597 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: poll_exit -2023-10-23 09:12:51,952 INFO SenderThread:54597 [sender.py:transition_state():626] send defer: 8 -2023-10-23 09:12:51,952 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: poll_exit -2023-10-23 09:12:51,952 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:12:51,952 INFO HandlerThread:54597 [handler.py:handle_request_defer():170] handle defer: 8 -2023-10-23 09:12:51,952 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: defer -2023-10-23 09:12:51,952 INFO SenderThread:54597 [sender.py:send_request_defer():622] handle sender defer: 8 -2023-10-23 09:12:51,962 INFO SenderThread:54597 [sender.py:transition_state():626] send defer: 9 -2023-10-23 09:12:51,962 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:12:51,962 DEBUG SenderThread:54597 [sender.py:send():375] send: artifact -2023-10-23 09:12:51,962 INFO HandlerThread:54597 [handler.py:handle_request_defer():170] handle defer: 9 -2023-10-23 09:12:52,169 INFO Thread-12 :54597 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/output.log -2023-10-23 09:12:52,598 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: poll_exit -2023-10-23 09:12:53,080 INFO SenderThread:54597 [sender.py:send_artifact():1474] sent artifact job-https___github.com_junchen14_MiniGPT-4_finetune_train.py - {'id': 'QXJ0aWZhY3Q6NjEzMjczODU5', 'digest': 'e1357229d7190882bf8b2dba86a37a56', 'state': 'COMMITTED', 'aliases': [{'artifactCollectionName': 'job-https___github.com_junchen14_MiniGPT-4_finetune_train.py', 'alias': 'latest'}, {'artifactCollectionName': 'job-https___github.com_junchen14_MiniGPT-4_finetune_train.py', 'alias': 'v0'}], 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjEwOTExOTI4MQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6NjEzMjczODU5', 'versionIndex': 0}}, 'version': 'v0'} -2023-10-23 09:12:53,080 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: defer -2023-10-23 09:12:53,080 INFO SenderThread:54597 [sender.py:send_request_defer():622] handle sender defer: 9 -2023-10-23 09:12:53,080 INFO SenderThread:54597 [dir_watcher.py:finish():365] shutting down directory watcher -2023-10-23 09:12:53,170 INFO SenderThread:54597 [dir_watcher.py:finish():395] scan: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files -2023-10-23 09:12:53,170 INFO SenderThread:54597 [dir_watcher.py:finish():409] scan save: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/config.yaml config.yaml -2023-10-23 09:12:53,170 INFO SenderThread:54597 [dir_watcher.py:finish():409] scan save: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/output.log output.log -2023-10-23 09:12:53,173 INFO SenderThread:54597 [dir_watcher.py:finish():409] scan save: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-metadata.json wandb-metadata.json -2023-10-23 09:12:53,173 INFO SenderThread:54597 [dir_watcher.py:finish():409] scan save: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/conda-environment.yaml conda-environment.yaml -2023-10-23 09:12:53,176 INFO SenderThread:54597 [dir_watcher.py:finish():409] scan save: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/requirements.txt requirements.txt -2023-10-23 09:12:53,178 INFO SenderThread:54597 [dir_watcher.py:finish():409] scan save: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json wandb-summary.json -2023-10-23 09:12:53,178 INFO SenderThread:54597 [sender.py:transition_state():626] send defer: 10 -2023-10-23 09:12:53,178 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: poll_exit -2023-10-23 09:12:53,180 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:12:53,183 INFO HandlerThread:54597 [handler.py:handle_request_defer():170] handle defer: 10 -2023-10-23 09:12:53,183 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: defer -2023-10-23 09:12:53,183 INFO SenderThread:54597 [sender.py:send_request_defer():622] handle sender defer: 10 -2023-10-23 09:12:53,183 INFO SenderThread:54597 [file_pusher.py:finish():168] shutting down file pusher -2023-10-23 09:12:53,598 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: poll_exit -2023-10-23 09:12:53,599 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: poll_exit -2023-10-23 09:12:53,859 INFO wandb-upload_2:54597 [upload_job.py:push():137] Uploaded file /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/conda-environment.yaml -2023-10-23 09:12:54,033 INFO wandb-upload_0:54597 [upload_job.py:push():137] Uploaded file /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/config.yaml -2023-10-23 09:12:54,177 INFO wandb-upload_1:54597 [upload_job.py:push():137] Uploaded file /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/output.log -2023-10-23 09:12:54,187 INFO wandb-upload_4:54597 [upload_job.py:push():137] Uploaded file /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/wandb-summary.json -2023-10-23 09:12:54,206 INFO wandb-upload_3:54597 [upload_job.py:push():137] Uploaded file /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/files/requirements.txt -2023-10-23 09:12:54,407 INFO Thread-11 :54597 [sender.py:transition_state():626] send defer: 11 -2023-10-23 09:12:54,407 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:12:54,407 INFO HandlerThread:54597 [handler.py:handle_request_defer():170] handle defer: 11 -2023-10-23 09:12:54,407 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: defer -2023-10-23 09:12:54,407 INFO SenderThread:54597 [sender.py:send_request_defer():622] handle sender defer: 11 -2023-10-23 09:12:54,407 INFO SenderThread:54597 [file_pusher.py:join():173] waiting for file pusher -2023-10-23 09:12:54,407 INFO SenderThread:54597 [sender.py:transition_state():626] send defer: 12 -2023-10-23 09:12:54,408 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:12:54,408 INFO HandlerThread:54597 [handler.py:handle_request_defer():170] handle defer: 12 -2023-10-23 09:12:54,408 DEBUG SenderThread:54597 [sender.py:send_request():402] send_request: defer -2023-10-23 09:12:54,408 INFO SenderThread:54597 [sender.py:send_request_defer():622] handle sender defer: 12 -2023-10-23 09:12:54,599 DEBUG HandlerThread:54597 [handler.py:handle_request():144] handle_request: poll_exit -2023-10-23 09:12:55,181 WARNING StreamThr :54597 [internal.py:is_dead():414] Internal process exiting, parent pid 53640 disappeared -2023-10-23 09:12:55,181 ERROR StreamThr :54597 [internal.py:wandb_internal():152] Internal process shutdown. -2023-10-23 09:12:55,599 INFO HandlerThread:54597 [handler.py:finish():845] shutting down handler -2023-10-23 09:12:55,599 INFO WriterThread:54597 [datastore.py:close():298] close: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/run-txn4i28h.wandb -2023-10-23 09:12:56,034 INFO SenderThread:54597 [sender.py:transition_state():626] send defer: 13 -2023-10-23 09:12:56,034 INFO SenderThread:54597 [sender.py:finish():1550] shutting down sender -2023-10-23 09:12:56,034 INFO SenderThread:54597 [file_pusher.py:finish():168] shutting down file pusher -2023-10-23 09:12:56,034 INFO SenderThread:54597 [file_pusher.py:join():173] waiting for file pusher diff --git a/wandb/run-20231023_091029-txn4i28h/logs/debug.log b/wandb/run-20231023_091029-txn4i28h/logs/debug.log deleted file mode 100644 index a67e3c3..0000000 --- a/wandb/run-20231023_091029-txn4i28h/logs/debug.log +++ /dev/null @@ -1,26 +0,0 @@ -2023-10-23 09:10:30,028 INFO MainThread:53640 [wandb_setup.py:_flush():76] Configure stats pid to 53640 -2023-10-23 09:10:30,028 INFO MainThread:53640 [wandb_setup.py:_flush():76] Loading settings from /home/chenj0g/.config/wandb/settings -2023-10-23 09:10:30,028 INFO MainThread:53640 [wandb_setup.py:_flush():76] Loading settings from /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/settings -2023-10-23 09:10:30,028 INFO MainThread:53640 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} -2023-10-23 09:10:30,028 INFO MainThread:53640 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'train.py', 'program': '/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/train.py'} -2023-10-23 09:10:30,028 INFO MainThread:53640 [wandb_init.py:_log_setup():507] Logging user logs to /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/logs/debug.log -2023-10-23 09:10:30,028 INFO MainThread:53640 [wandb_init.py:_log_setup():508] Logging internal logs to /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091029-txn4i28h/logs/debug-internal.log -2023-10-23 09:10:30,028 INFO MainThread:53640 [wandb_init.py:init():547] calling init triggers -2023-10-23 09:10:30,029 INFO MainThread:53640 [wandb_init.py:init():554] wandb.init called with sweep_config: {} -config: {} -2023-10-23 09:10:30,029 INFO MainThread:53640 [wandb_init.py:init():595] starting backend -2023-10-23 09:10:30,029 INFO MainThread:53640 [wandb_init.py:init():599] setting up manager -2023-10-23 09:10:30,030 INFO MainThread:53640 [backend.py:_multiprocessing_setup():106] multiprocessing start_methods=fork,spawn,forkserver, using: spawn -2023-10-23 09:10:30,035 INFO MainThread:53640 [wandb_init.py:init():605] backend started and connected -2023-10-23 09:10:30,037 INFO MainThread:53640 [wandb_init.py:init():695] updated telemetry -2023-10-23 09:10:30,446 INFO MainThread:53640 [wandb_init.py:init():732] communicating run to backend with 60.0 second timeout -2023-10-23 09:10:31,053 INFO MainThread:53640 [wandb_run.py:_on_init():2176] communicating current version -2023-10-23 09:10:31,357 INFO MainThread:53640 [wandb_run.py:_on_init():2185] got version response upgrade_message: "wandb version 0.15.12 is available! To upgrade, please run:\n $ pip install wandb --upgrade" - -2023-10-23 09:10:31,357 INFO MainThread:53640 [wandb_init.py:init():782] starting run threads in backend -2023-10-23 09:10:37,065 INFO MainThread:53640 [wandb_run.py:_console_start():2157] atexit reg -2023-10-23 09:10:37,065 INFO MainThread:53640 [wandb_run.py:_redirect():2012] redirect: SettingsConsole.WRAP_RAW -2023-10-23 09:10:37,065 INFO MainThread:53640 [wandb_run.py:_redirect():2077] Wrapping output streams. -2023-10-23 09:10:37,065 INFO MainThread:53640 [wandb_run.py:_redirect():2102] Redirects installed. -2023-10-23 09:10:37,066 INFO MainThread:53640 [wandb_init.py:init():824] run started, returning control to user process -2023-10-23 09:10:37,066 INFO MainThread:53640 [wandb_watch.py:watch():51] Watching diff --git a/wandb/run-20231023_091029-txn4i28h/run-txn4i28h.wandb b/wandb/run-20231023_091029-txn4i28h/run-txn4i28h.wandb deleted file mode 100644 index 313e061..0000000 Binary files a/wandb/run-20231023_091029-txn4i28h/run-txn4i28h.wandb and /dev/null differ diff --git a/wandb/run-20231023_091030-25yuqsbo/files/conda-environment.yaml b/wandb/run-20231023_091030-25yuqsbo/files/conda-environment.yaml deleted file mode 100644 index cfced9e..0000000 --- a/wandb/run-20231023_091030-25yuqsbo/files/conda-environment.yaml +++ /dev/null @@ -1,297 +0,0 @@ -name: eye -channels: - - pytorch - - defaults -dependencies: - - _libgcc_mutex=0.1=main - - _openmp_mutex=5.1=1_gnu - - blas=1.0=mkl - - brotlipy=0.7.0=py39h27cfd23_1003 - - bzip2=1.0.8=h7b6447c_0 - - ca-certificates=2023.01.10=h06a4308_0 - - certifi=2022.12.7=py39h06a4308_0 - - cffi=1.15.1=py39h5eee18b_3 - - charset-normalizer=2.0.4=pyhd3eb1b0_0 - - cryptography=39.0.1=py39h9ce1e76_0 - - cudatoolkit=11.3.1=h2bc3f7f_2 - - ffmpeg=4.3=hf484d3e_0 - - flit-core=3.8.0=py39h06a4308_0 - - freetype=2.12.1=h4a9f257_0 - - giflib=5.2.1=h5eee18b_3 - - gmp=6.2.1=h295c915_3 - - gnutls=3.6.15=he1e5248_0 - - idna=3.4=py39h06a4308_0 - - intel-openmp=2021.4.0=h06a4308_3561 - - jpeg=9e=h5eee18b_1 - - lame=3.100=h7b6447c_0 - - lcms2=2.12=h3be6417_0 - - ld_impl_linux-64=2.38=h1181459_1 - - lerc=3.0=h295c915_0 - - libdeflate=1.17=h5eee18b_0 - - libffi=3.4.2=h6a678d5_6 - - libgcc-ng=11.2.0=h1234567_1 - - libgomp=11.2.0=h1234567_1 - - libiconv=1.16=h7f8727e_2 - - libidn2=2.3.2=h7f8727e_0 - - libpng=1.6.39=h5eee18b_0 - - libstdcxx-ng=11.2.0=h1234567_1 - - libtasn1=4.19.0=h5eee18b_0 - - libtiff=4.5.0=h6a678d5_2 - - libunistring=0.9.10=h27cfd23_0 - - libwebp=1.2.4=h11a3e52_1 - - libwebp-base=1.2.4=h5eee18b_1 - - lz4-c=1.9.4=h6a678d5_0 - - mkl=2021.4.0=h06a4308_640 - - mkl-service=2.4.0=py39h7f8727e_0 - - mkl_fft=1.3.1=py39hd3c417c_0 - - mkl_random=1.2.2=py39h51133e4_0 - - ncurses=6.4=h6a678d5_0 - - nettle=3.7.3=hbbd107a_1 - - numpy=1.23.5=py39h14f4228_0 - - numpy-base=1.23.5=py39h31eccc5_0 - - openh264=2.1.1=h4ff587b_0 - - openssl=1.1.1t=h7f8727e_0 - - pillow=9.4.0=py39h6a678d5_0 - - pip=23.0.1=py39h06a4308_0 - - pycparser=2.21=pyhd3eb1b0_0 - - pyopenssl=23.0.0=py39h06a4308_0 - - pysocks=1.7.1=py39h06a4308_0 - - python=3.9.16=h7a1cb2a_2 - - pytorch-mutex=1.0=cuda - - readline=8.2=h5eee18b_0 - - requests=2.28.1=py39h06a4308_1 - - setuptools=65.6.3=py39h06a4308_0 - - six=1.16.0=pyhd3eb1b0_1 - - sqlite=3.41.1=h5eee18b_0 - - tk=8.6.12=h1ccaba5_0 - - torchaudio=0.12.1=py39_cu113 - - torchvision=0.13.1=py39_cu113 - - typing_extensions=4.4.0=py39h06a4308_0 - - urllib3=1.26.15=py39h06a4308_0 - - wheel=0.38.4=py39h06a4308_0 - - xz=5.2.10=h5eee18b_1 - - zlib=1.2.13=h5eee18b_0 - - zstd=1.5.4=hc292b87_0 - - pip: - - accelerate==0.20.3 - - aiofiles==23.1.0 - - aiohttp==3.8.4 - - aiosignal==1.3.1 - - altair==4.2.2 - - antlr4-python3-runtime==4.9.3 - - anyio==3.6.2 - - appdirs==1.4.4 - - argon2-cffi==21.3.0 - - argon2-cffi-bindings==21.2.0 - - arrow==1.2.3 - - asttokens==2.2.1 - - async-timeout==4.0.2 - - attrs==22.2.0 - - backcall==0.2.0 - - beautifulsoup4==4.12.2 - - bitsandbytes==0.37.0 - - bleach==6.0.0 - - blis==0.7.9 - - braceexpand==0.1.7 - - catalogue==2.0.8 - - cchardet==2.1.7 - - chardet==5.1.0 - - click==8.1.3 - - cloudpickle==2.2.1 - - cmake==3.26.1 - - comm==0.1.3 - - confection==0.0.4 - - contourpy==1.0.7 - - cycler==0.11.0 - - cymem==2.0.7 - - datasets==2.13.1 - - debugpy==1.6.7 - - decorator==5.1.1 - - decord==0.6.0 - - defusedxml==0.7.1 - - dill==0.3.6 - - docker-pycreds==0.4.0 - - einops==0.6.0 - - entrypoints==0.4 - - executing==1.2.0 - - fairscale==0.4.13 - - fastapi==0.95.0 - - fastjsonschema==2.16.3 - - ffmpy==0.3.0 - - filelock==3.9.0 - - fire==0.5.0 - - fonttools==4.38.0 - - fqdn==1.5.1 - - frozenlist==1.3.3 - - fsspec==2023.4.0 - - gitdb==4.0.10 - - gitpython==3.1.31 - - gradio==3.24.1 - - gradio-client==0.0.8 - - h11==0.14.0 - - httpcore==0.16.3 - - httpx==0.23.3 - - huggingface-hub==0.16.4 - - imageio==2.31.1 - - importlib-metadata==6.2.0 - - importlib-resources==5.12.0 - - iopath==0.1.10 - - ipykernel==6.22.0 - - ipython==8.12.0 - - ipython-genutils==0.2.0 - - isoduration==20.11.0 - - jedi==0.18.2 - - jinja2==3.1.2 - - joblib==1.2.0 - - jsonpointer==2.3 - - jsonschema==4.17.3 - - jupyter-client==8.1.0 - - jupyter-core==5.3.0 - - jupyter-events==0.6.3 - - jupyter-server==2.5.0 - - jupyter-server-terminals==0.4.4 - - jupyterlab-pygments==0.2.2 - - kiwisolver==1.4.4 - - langcodes==3.3.0 - - lazy-loader==0.2 - - linkify-it-py==2.0.0 - - lit==16.0.0 - - llvmlite==0.39.1 - - loralib==0.1.1 - - markdown-it-py==2.2.0 - - markupsafe==2.1.2 - - matplotlib==3.7.0 - - matplotlib-inline==0.1.6 - - mdit-py-plugins==0.3.3 - - mdurl==0.1.2 - - mistune==2.0.5 - - mpmath==1.3.0 - - multidict==6.0.4 - - multiprocess==0.70.14 - - murmurhash==1.0.9 - - nbclassic==0.5.5 - - nbclient==0.7.3 - - nbconvert==7.3.0 - - nbformat==5.8.0 - - nest-asyncio==1.5.6 - - networkx==3.1 - - nltk==3.8.1 - - notebook==6.5.4 - - notebook-shim==0.2.2 - - numba==0.56.4 - - nvidia-cublas-cu11==11.10.3.66 - - nvidia-cuda-cupti-cu11==11.7.101 - - nvidia-cuda-nvrtc-cu11==11.7.99 - - nvidia-cuda-runtime-cu11==11.7.99 - - nvidia-cudnn-cu11==8.5.0.96 - - nvidia-cufft-cu11==10.9.0.58 - - nvidia-curand-cu11==10.2.10.91 - - nvidia-cusolver-cu11==11.4.0.1 - - nvidia-cusparse-cu11==11.7.4.91 - - nvidia-nccl-cu11==2.14.3 - - nvidia-nvtx-cu11==11.7.91 - - omegaconf==2.3.0 - - openai==0.27.0 - - opencv-python==4.7.0.72 - - orjson==3.8.9 - - packaging==23.0 - - pandas==2.0.0 - - pandocfilters==1.5.0 - - parso==0.8.3 - - pathlib==1.0.1 - - pathtools==0.1.2 - - pathy==0.10.1 - - peft==0.2.0 - - pexpect==4.8.0 - - pickleshare==0.7.5 - - platformdirs==3.2.0 - - policies==0.4.2 - - portalocker==2.7.0 - - preshed==3.0.8 - - progressbar2==4.2.0 - - prometheus-client==0.16.0 - - prompt-toolkit==3.0.38 - - protobuf==4.22.1 - - psutil==5.9.4 - - ptyprocess==0.7.0 - - pure-eval==0.2.2 - - pyarrow==12.0.1 - - pycocoevalcap==1.2 - - pycocotools==2.0.6 - - pycryptodome==3.17 - - pydantic==1.10.7 - - pydub==0.25.1 - - pygments==2.14.0 - - pynndescent==0.5.8 - - pyparsing==3.0.9 - - pyrsistent==0.19.3 - - python-dateutil==2.8.2 - - python-json-logger==2.0.7 - - python-multipart==0.0.6 - - python-utils==3.7.0 - - pytz==2023.3 - - pywavelets==1.4.1 - - pyyaml==6.0 - - pyzmq==25.0.2 - - regex==2022.10.31 - - rfc3339-validator==0.1.4 - - rfc3986==1.5.0 - - rfc3986-validator==0.1.1 - - safetensors==0.3.1 - - scikit-image==0.21.0 - - scikit-learn==1.2.2 - - scipy==1.10.1 - - seaborn==0.12.2 - - semantic-version==2.10.0 - - send2trash==1.8.0 - - sentence-transformers==2.2.2 - - sentencepiece==0.1.97 - - sentry-sdk==1.19.1 - - setproctitle==1.3.2 - - smart-open==6.3.0 - - smmap==5.0.0 - - sniffio==1.3.0 - - soupsieve==2.4 - - spacy==3.5.1 - - spacy-legacy==3.0.12 - - spacy-loggers==1.0.4 - - srsly==2.4.6 - - stack-data==0.6.2 - - starlette==0.26.1 - - submitit==1.4.5 - - sympy==1.11.1 - - tenacity==8.2.2 - - termcolor==2.3.0 - - terminado==0.17.1 - - thinc==8.1.9 - - threadpoolctl==3.1.0 - - tifffile==2023.4.12 - - timm==0.6.13 - - tinycss2==1.2.1 - - tokenizers==0.13.3 - - toolz==0.12.0 - - torch==2.0.0 - - tornado==6.2 - - tqdm==4.64.1 - - traitlets==5.9.0 - - transformers==4.31.0 - - triton==2.0.0 - - typer==0.7.0 - - tzdata==2023.3 - - uc-micro-py==1.0.1 - - umap-learn==0.5.3 - - uri-template==1.2.0 - - uvicorn==0.21.1 - - visual-genome==1.1.1 - - wandb==0.14.2 - - wasabi==1.1.1 - - wcwidth==0.2.6 - - webcolors==1.13 - - webdataset==0.2.48 - - webencodings==0.5.1 - - websocket-client==1.5.1 - - websockets==11.0.1 - - xxhash==3.2.0 - - yarl==1.8.2 - - zipp==3.14.0 -prefix: /home/chenj0g/anaconda3/envs/eye diff --git a/wandb/run-20231023_091030-25yuqsbo/files/config.yaml b/wandb/run-20231023_091030-25yuqsbo/files/config.yaml deleted file mode 100644 index a263d39..0000000 --- a/wandb/run-20231023_091030-25yuqsbo/files/config.yaml +++ /dev/null @@ -1,38 +0,0 @@ -wandb_version: 1 - -_wandb: - desc: null - value: - python_version: 3.9.16 - cli_version: 0.14.2 - framework: huggingface - huggingface_version: 4.31.0 - is_jupyter_run: false - is_kaggle_kernel: false - start_time: 1698041430.036499 - t: - 1: - - 1 - - 11 - - 41 - - 49 - - 55 - - 63 - - 71 - 2: - - 1 - - 11 - - 41 - - 49 - - 55 - - 63 - - 71 - 3: - - 1 - - 13 - - 23 - 4: 3.9.16 - 5: 0.14.2 - 6: 4.31.0 - 8: - - 5 diff --git a/wandb/run-20231023_091030-25yuqsbo/files/output.log b/wandb/run-20231023_091030-25yuqsbo/files/output.log deleted file mode 100644 index 2f6a9a3..0000000 --- a/wandb/run-20231023_091030-25yuqsbo/files/output.log +++ /dev/null @@ -1,161 +0,0 @@ - -2023-10-23 09:10:36,980 [INFO] Start training -batch sizes [[2]] -module.llama_model.base_model.model.model.layers.0.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.0.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.0.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.0.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.1.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.1.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.1.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.1.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.2.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.2.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.2.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.2.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.3.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.3.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.3.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.3.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.4.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.4.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.4.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.4.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.5.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.5.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.5.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.5.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.6.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.6.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.6.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.6.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.7.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.7.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.7.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.7.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.8.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.8.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.8.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.8.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.9.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.9.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.9.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.9.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.10.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.10.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.10.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.10.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.11.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.11.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.11.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.11.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.12.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.12.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.12.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.12.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.13.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.13.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.13.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.13.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.14.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.14.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.14.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.14.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.15.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.15.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.15.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.15.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.16.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.16.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.16.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.16.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.17.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.17.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.17.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.17.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.18.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.18.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.18.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.18.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.19.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.19.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.19.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.19.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.20.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.20.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.20.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.20.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.21.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.21.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.21.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.21.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.22.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.22.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.22.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.22.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.23.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.23.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.23.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.23.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.24.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.24.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.24.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.24.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.25.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.25.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.25.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.25.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.26.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.26.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.26.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.26.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.27.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.27.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.27.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.27.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.28.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.28.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.28.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.28.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.29.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.29.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.29.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.29.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.30.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.30.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.30.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.30.self_attn.v_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.31.self_attn.q_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.31.self_attn.q_proj.lora_B.weight -module.llama_model.base_model.model.model.layers.31.self_attn.v_proj.lora_A.weight -module.llama_model.base_model.model.model.layers.31.self_attn.v_proj.lora_B.weight -module.llama_proj.weight -module.llama_proj.bias -2023-10-23 09:10:39,273 [INFO] dataset_ratios not specified, datasets will be concatenated (map-style datasets) or chained (webdataset.DataPipeline). -2023-10-23 09:10:39,273 [INFO] Loaded 12171 records for train split from the dataset. -2023-10-23 09:10:39,291 [INFO] number of trainable parameters: 56627200 -2023-10-23 09:10:39,292 [INFO] Start training epoch 0, 1000 iters per inner epoch. -Train: data epoch: [0] [ 0/1000] eta: 0:54:33 lr: 0.000001 loss: 1.4162 time: 3.2736 data: 0.0000 max mem: 33043 -/home/chenj0g/anaconda3/envs/eye/lib/python3.9/site-packages/torch/utils/checkpoint.py:31: UserWarning: None of the inputs have requires_grad=True. Gradients will be None - warnings.warn("None of the inputs have requires_grad=True. Gradients will be None") -Train: data epoch: [0] [ 50/1000] eta: 0:08:41 lr: 0.000001 loss: 1.4426 time: 0.4963 data: 0.0000 max mem: 38128 -Train: data epoch: [0] [ 100/1000] eta: 0:07:50 lr: 0.000002 loss: 1.3406 time: 0.4881 data: 0.0000 max mem: 40846 -Train: data epoch: [0] [ 150/1000] eta: 0:07:14 lr: 0.000002 loss: 1.5900 time: 0.4890 data: 0.0000 max mem: 41228 -Train: data epoch: [0] [ 200/1000] eta: 0:06:45 lr: 0.000003 loss: 1.6308 time: 0.5138 data: 0.0000 max mem: 41228 -Train: data epoch: [0] [ 250/1000] eta: 0:06:18 lr: 0.000003 loss: 1.4190 time: 0.5009 data: 0.0000 max mem: 41228 -Traceback (most recent call last): - File "/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/train.py", line 113, in - File "/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/train.py", line 109, in main - File "/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/minigpt4/runners/runner_base.py", line 377, in train - train_stats = self.train_epoch(cur_epoch) - File "/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/minigpt4/runners/runner_base.py", line 437, in train_epoch - return self.task.train_epoch( - File "/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/minigpt4/tasks/base_task.py", line 116, in train_epoch - return self._train_inner_loop( - File "/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/minigpt4/tasks/base_task.py", line 225, in _train_inner_loop - scaler.scale(loss).backward() - File "/home/chenj0g/anaconda3/envs/eye/lib/python3.9/site-packages/torch/_tensor.py", line 487, in backward - torch.autograd.backward( - File "/home/chenj0g/anaconda3/envs/eye/lib/python3.9/site-packages/torch/autograd/__init__.py", line 200, in backward - Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass -KeyboardInterrupt \ No newline at end of file diff --git a/wandb/run-20231023_091030-25yuqsbo/files/requirements.txt b/wandb/run-20231023_091030-25yuqsbo/files/requirements.txt deleted file mode 100644 index 75bc393..0000000 --- a/wandb/run-20231023_091030-25yuqsbo/files/requirements.txt +++ /dev/null @@ -1,246 +0,0 @@ -accelerate==0.20.3 -aiofiles==23.2.1 -aiohttp==3.8.4 -aiosignal==1.3.1 -altair==5.1.2 -antlr4-python3-runtime==4.9.3 -anyio==3.6.2 -appdirs==1.4.4 -argon2-cffi-bindings==21.2.0 -argon2-cffi==21.3.0 -arrow==1.2.3 -asttokens==2.2.1 -async-timeout==4.0.2 -attrs==22.2.0 -backcall==0.2.0 -beautifulsoup4==4.12.2 -bitsandbytes==0.37.0 -bleach==6.0.0 -blis==0.7.9 -braceexpand==0.1.7 -brotlipy==0.7.0 -catalogue==2.0.8 -cchardet==2.1.7 -certifi==2022.12.7 -cffi==1.15.1 -chardet==5.1.0 -charset-normalizer==2.0.4 -click==8.1.3 -cloudpickle==2.2.1 -cmake==3.26.1 -comm==0.1.3 -confection==0.0.4 -contourpy==1.0.7 -cryptography==39.0.1 -cycler==0.11.0 -cymem==2.0.7 -datasets==2.13.1 -debugpy==1.6.7 -decorator==5.1.1 -decord==0.6.0 -defusedxml==0.7.1 -dill==0.3.6 -docker-pycreds==0.4.0 -einops==0.6.0 -entrypoints==0.4 -executing==1.2.0 -fairscale==0.4.13 -fastapi==0.95.0 -fastjsonschema==2.16.3 -ffmpy==0.3.1 -filelock==3.9.0 -fire==0.5.0 -flit-core==3.8.0 -fonttools==4.38.0 -fqdn==1.5.1 -frozenlist==1.3.3 -fsspec==2023.9.2 -gitdb==4.0.10 -gitpython==3.1.31 -gradio-client==0.6.0 -gradio==3.24.1 -h11==0.14.0 -httpcore==0.18.0 -httpx==0.25.0 -huggingface-hub==0.18.0 -idna==3.4 -imageio==2.31.1 -importlib-metadata==6.2.0 -importlib-resources==5.12.0 -iopath==0.1.10 -ipykernel==6.22.0 -ipython-genutils==0.2.0 -ipython==8.12.0 -isoduration==20.11.0 -jedi==0.18.2 -jinja2==3.1.2 -joblib==1.2.0 -jsonpointer==2.3 -jsonschema==4.17.3 -jupyter-client==8.1.0 -jupyter-core==5.3.0 -jupyter-events==0.6.3 -jupyter-server-terminals==0.4.4 -jupyter-server==2.5.0 -jupyterlab-pygments==0.2.2 -kiwisolver==1.4.4 -langcodes==3.3.0 -lazy-loader==0.2 -linkify-it-py==2.0.2 -lit==16.0.0 -llvmlite==0.39.1 -loralib==0.1.1 -markdown-it-py==2.2.0 -markupsafe==2.1.2 -matplotlib-inline==0.1.6 -matplotlib==3.7.0 -mdit-py-plugins==0.3.3 -mdurl==0.1.2 -mistune==2.0.5 -mkl-fft==1.3.1 -mkl-random==1.2.2 -mkl-service==2.4.0 -mpmath==1.3.0 -multidict==6.0.4 -multiprocess==0.70.14 -murmurhash==1.0.9 -nbclassic==0.5.5 -nbclient==0.7.3 -nbconvert==7.3.0 -nbformat==5.8.0 -nest-asyncio==1.5.6 -networkx==3.1 -nltk==3.8.1 -notebook-shim==0.2.2 -notebook==6.5.4 -numba==0.56.4 -numpy==1.23.5 -nvidia-cublas-cu11==11.10.3.66 -nvidia-cuda-cupti-cu11==11.7.101 -nvidia-cuda-nvrtc-cu11==11.7.99 -nvidia-cuda-runtime-cu11==11.7.99 -nvidia-cudnn-cu11==8.5.0.96 -nvidia-cufft-cu11==10.9.0.58 -nvidia-curand-cu11==10.2.10.91 -nvidia-cusolver-cu11==11.4.0.1 -nvidia-cusparse-cu11==11.7.4.91 -nvidia-nccl-cu11==2.14.3 -nvidia-nvtx-cu11==11.7.91 -omegaconf==2.3.0 -openai==0.27.0 -opencv-python==4.7.0.72 -orjson==3.9.8 -packaging==23.0 -pandas==2.0.0 -pandocfilters==1.5.0 -parso==0.8.3 -pathlib==1.0.1 -pathtools==0.1.2 -pathy==0.10.1 -peft==0.2.0 -pexpect==4.8.0 -pickleshare==0.7.5 -pillow==9.4.0 -pip==23.0.1 -platformdirs==3.2.0 -policies==0.4.2 -portalocker==2.7.0 -preshed==3.0.8 -progressbar2==4.2.0 -prometheus-client==0.16.0 -prompt-toolkit==3.0.38 -protobuf==4.22.1 -psutil==5.9.4 -ptyprocess==0.7.0 -pure-eval==0.2.2 -pyarrow==12.0.1 -pycocoevalcap==1.2 -pycocotools==2.0.6 -pycparser==2.21 -pycryptodome==3.17 -pydantic==1.10.7 -pydub==0.25.1 -pygments==2.14.0 -pynndescent==0.5.8 -pyopenssl==23.0.0 -pyparsing==3.0.9 -pyrsistent==0.19.3 -pysocks==1.7.1 -python-dateutil==2.8.2 -python-json-logger==2.0.7 -python-multipart==0.0.6 -python-utils==3.7.0 -pytz==2023.3 -pywavelets==1.4.1 -pyyaml==6.0 -pyzmq==25.0.2 -regex==2022.10.31 -requests==2.28.1 -rfc3339-validator==0.1.4 -rfc3986-validator==0.1.1 -rfc3986==1.5.0 -rpe-index==1.2.0 -safetensors==0.3.1 -scikit-image==0.21.0 -scikit-learn==1.2.2 -scipy==1.10.1 -seaborn==0.12.2 -semantic-version==2.10.0 -send2trash==1.8.0 -sentence-transformers==2.2.2 -sentencepiece==0.1.97 -sentry-sdk==1.19.1 -setproctitle==1.3.2 -setuptools==65.6.3 -six==1.16.0 -smart-open==6.3.0 -smmap==5.0.0 -sniffio==1.3.0 -soupsieve==2.4 -spacy-legacy==3.0.12 -spacy-loggers==1.0.4 -spacy==3.5.1 -srsly==2.4.6 -stack-data==0.6.2 -starlette==0.26.1 -submitit==1.4.5 -sympy==1.11.1 -tenacity==8.2.2 -termcolor==2.3.0 -terminado==0.17.1 -thinc==8.1.9 -threadpoolctl==3.1.0 -tifffile==2023.4.12 -timm==0.6.13 -tinycss2==1.2.1 -tokenizers==0.13.3 -toolz==0.12.0 -torch==2.0.0 -torchaudio==0.12.1 -torchvision==0.13.1 -tornado==6.2 -tqdm==4.64.1 -traitlets==5.9.0 -transformers==4.31.0 -triton==2.0.0 -typer==0.7.0 -typing-extensions==4.4.0 -tzdata==2023.3 -uc-micro-py==1.0.2 -umap-learn==0.5.3 -uri-template==1.2.0 -urllib3==1.26.15 -uvicorn==0.21.1 -visual-genome==1.1.1 -wandb==0.14.2 -wasabi==1.1.1 -wcwidth==0.2.6 -webcolors==1.13 -webdataset==0.2.48 -webencodings==0.5.1 -websocket-client==1.5.1 -websockets==11.0.1 -wheel==0.38.4 -xxhash==3.2.0 -yarl==1.8.2 -zipp==3.14.0 \ No newline at end of file diff --git a/wandb/run-20231023_091030-25yuqsbo/files/wandb-metadata.json b/wandb/run-20231023_091030-25yuqsbo/files/wandb-metadata.json deleted file mode 100644 index 1511845..0000000 --- a/wandb/run-20231023_091030-25yuqsbo/files/wandb-metadata.json +++ /dev/null @@ -1,374 +0,0 @@ -{ - "os": "Linux-3.10.0-1160.76.1.el7.x86_64-x86_64-with-glibc2.17", - "python": "3.9.16", - "heartbeatAt": "2023-10-23T06:10:31.385390", - "startedAt": "2023-10-23T06:10:30.007205", - "docker": null, - "cuda": null, - "args": [ - "--job_name", - "minigpt4_v2_test", - "--cfg-path", - "train_configs/minigpt_v2_finetune.yaml" - ], - "state": "running", - "program": "/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/train.py", - "codePath": "train.py", - "git": { - "remote": "https://github.com/junchen14/MiniGPT-4_finetune", - "commit": "5c3ec8bb730b2e9df23b4bb3eb24108f9c04a8d6" - }, - "email": "jun.chen@kaust.edu.sa", - "root": "/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune", - "host": "gpu201-16-l", - "username": "chenj0g", - "executable": "/home/chenj0g/anaconda3/envs/eye/bin/python", - "cpu_count": 64, - "cpu_count_logical": 64, - "cpu_freq": { - "current": 1996.2770000000016, - "min": 0.0, - "max": 0.0 - }, - "cpu_freq_per_core": [ - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - }, - { - "current": 1996.277, - "min": 0.0, - "max": 0.0 - } - ], - "disk": { - "total": 251.85773849487305, - "used": 7.905696868896484 - }, - "gpu": "NVIDIA A100-SXM4-80GB", - "gpu_count": 2, - "gpu_devices": [ - { - "name": "NVIDIA A100-SXM4-80GB", - "memory_total": 85899345920 - }, - { - "name": "NVIDIA A100-SXM4-80GB", - "memory_total": 85899345920 - } - ], - "memory": { - "total": 503.7154769897461 - } -} diff --git a/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json b/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json deleted file mode 100644 index 4902901..0000000 --- a/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json +++ /dev/null @@ -1 +0,0 @@ -{"epoch": 0, "loss": 1.2293027639389038, "_timestamp": 1698041570.1777537, "_runtime": 140.1412546634674, "_step": 258, "_wandb": {"runtime": 139}} \ No newline at end of file diff --git a/wandb/run-20231023_091030-25yuqsbo/logs/debug-internal.log b/wandb/run-20231023_091030-25yuqsbo/logs/debug-internal.log deleted file mode 100644 index 5f307dc..0000000 --- a/wandb/run-20231023_091030-25yuqsbo/logs/debug-internal.log +++ /dev/null @@ -1,1367 +0,0 @@ -2023-10-23 09:10:30,037 INFO StreamThr :54598 [internal.py:wandb_internal():86] W&B internal server running at pid: 54598, started at: 2023-10-23 09:10:30.036505 -2023-10-23 09:10:30,038 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: status -2023-10-23 09:10:30,038 INFO WriterThread:54598 [datastore.py:open_for_write():85] open: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/run-25yuqsbo.wandb -2023-10-23 09:10:30,040 DEBUG SenderThread:54598 [sender.py:send():375] send: header -2023-10-23 09:10:30,457 DEBUG SenderThread:54598 [sender.py:send():375] send: run -2023-10-23 09:10:31,051 INFO SenderThread:54598 [dir_watcher.py:__init__():219] watching files in: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files -2023-10-23 09:10:31,051 INFO SenderThread:54598 [sender.py:_start_run_threads():1124] run started: 25yuqsbo with start time 1698041430.036499 -2023-10-23 09:10:31,052 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:31,052 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:31,053 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: check_version -2023-10-23 09:10:31,053 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: check_version -2023-10-23 09:10:31,364 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: run_start -2023-10-23 09:10:31,369 DEBUG HandlerThread:54598 [system_info.py:__init__():31] System info init -2023-10-23 09:10:31,369 DEBUG HandlerThread:54598 [system_info.py:__init__():46] System info init done -2023-10-23 09:10:31,369 INFO HandlerThread:54598 [system_monitor.py:start():181] Starting system monitor -2023-10-23 09:10:31,369 INFO SystemMonitor:54598 [system_monitor.py:_start():145] Starting system asset monitoring threads -2023-10-23 09:10:31,369 INFO HandlerThread:54598 [system_monitor.py:probe():201] Collecting system info -2023-10-23 09:10:31,370 INFO SystemMonitor:54598 [interfaces.py:start():190] Started cpu monitoring -2023-10-23 09:10:31,370 INFO SystemMonitor:54598 [interfaces.py:start():190] Started disk monitoring -2023-10-23 09:10:31,371 INFO SystemMonitor:54598 [interfaces.py:start():190] Started gpu monitoring -2023-10-23 09:10:31,371 INFO SystemMonitor:54598 [interfaces.py:start():190] Started memory monitoring -2023-10-23 09:10:31,371 INFO SystemMonitor:54598 [interfaces.py:start():190] Started network monitoring -2023-10-23 09:10:31,385 DEBUG HandlerThread:54598 [system_info.py:probe():195] Probing system -2023-10-23 09:10:31,390 DEBUG HandlerThread:54598 [system_info.py:_probe_git():180] Probing git -2023-10-23 09:10:31,403 DEBUG HandlerThread:54598 [system_info.py:_probe_git():188] Probing git done -2023-10-23 09:10:31,403 DEBUG HandlerThread:54598 [system_info.py:probe():240] Probing system done -2023-10-23 09:10:31,404 DEBUG HandlerThread:54598 [system_monitor.py:probe():210] {'os': 'Linux-3.10.0-1160.76.1.el7.x86_64-x86_64-with-glibc2.17', 'python': '3.9.16', 'heartbeatAt': '2023-10-23T06:10:31.385390', 'startedAt': '2023-10-23T06:10:30.007205', 'docker': None, 'cuda': None, 'args': ('--job_name', 'minigpt4_v2_test', '--cfg-path', 'train_configs/minigpt_v2_finetune.yaml'), 'state': 'running', 'program': '/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/train.py', 'codePath': 'train.py', 'git': {'remote': 'https://github.com/junchen14/MiniGPT-4_finetune', 'commit': '5c3ec8bb730b2e9df23b4bb3eb24108f9c04a8d6'}, 'email': 'jun.chen@kaust.edu.sa', 'root': '/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune', 'host': 'gpu201-16-l', 'username': 'chenj0g', 'executable': '/home/chenj0g/anaconda3/envs/eye/bin/python', 'cpu_count': 64, 'cpu_count_logical': 64, 'cpu_freq': {'current': 1996.2770000000016, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}, {'current': 1996.277, 'min': 0.0, 'max': 0.0}], 'disk': {'total': 251.85773849487305, 'used': 7.905696868896484}, 'gpu': 'NVIDIA A100-SXM4-80GB', 'gpu_count': 2, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}], 'memory': {'total': 503.7154769897461}} -2023-10-23 09:10:31,404 INFO HandlerThread:54598 [system_monitor.py:probe():211] Finished collecting system info -2023-10-23 09:10:31,404 INFO HandlerThread:54598 [system_monitor.py:probe():214] Publishing system info -2023-10-23 09:10:31,404 DEBUG HandlerThread:54598 [system_info.py:_save_pip():51] Saving list of pip packages installed into the current environment -2023-10-23 09:10:31,405 DEBUG HandlerThread:54598 [system_info.py:_save_pip():67] Saving pip packages done -2023-10-23 09:10:31,405 DEBUG HandlerThread:54598 [system_info.py:_save_conda():74] Saving list of conda packages installed into the current environment -2023-10-23 09:10:32,054 INFO Thread-12 :54598 [dir_watcher.py:_on_file_created():278] file/dir created: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/conda-environment.yaml -2023-10-23 09:10:32,054 INFO Thread-12 :54598 [dir_watcher.py:_on_file_created():278] file/dir created: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:10:32,054 INFO Thread-12 :54598 [dir_watcher.py:_on_file_created():278] file/dir created: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/requirements.txt -2023-10-23 09:10:36,957 DEBUG HandlerThread:54598 [system_info.py:_save_conda():86] Saving conda packages done -2023-10-23 09:10:36,959 INFO HandlerThread:54598 [system_monitor.py:probe():216] Finished publishing system info -2023-10-23 09:10:36,965 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:10:36,965 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: keepalive -2023-10-23 09:10:36,965 DEBUG SenderThread:54598 [sender.py:send():375] send: files -2023-10-23 09:10:36,965 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-metadata.json with policy now -2023-10-23 09:10:36,972 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 09:10:36,972 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: stop_status -2023-10-23 09:10:37,056 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/conda-environment.yaml -2023-10-23 09:10:37,057 INFO Thread-12 :54598 [dir_watcher.py:_on_file_created():278] file/dir created: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-metadata.json -2023-10-23 09:10:37,315 DEBUG SenderThread:54598 [sender.py:send():375] send: telemetry -2023-10-23 09:10:37,316 DEBUG SenderThread:54598 [sender.py:send():375] send: telemetry -2023-10-23 09:10:38,057 INFO Thread-12 :54598 [dir_watcher.py:_on_file_created():278] file/dir created: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/output.log -2023-10-23 09:10:38,321 INFO wandb-upload_0:54598 [upload_job.py:push():137] Uploaded file /tmp/tmp2rfhjpecwandb/l4nz4hzg-wandb-metadata.json -2023-10-23 09:10:40,059 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/output.log -2023-10-23 09:10:41,298 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:10:42,060 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/output.log -2023-10-23 09:10:42,566 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:42,567 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:42,567 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:42,568 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:43,061 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:10:43,093 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:43,094 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:43,094 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:43,095 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:43,680 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:43,681 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:43,681 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:43,682 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:44,062 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/output.log -2023-10-23 09:10:44,062 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:10:44,147 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:44,147 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:44,147 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:44,148 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:44,652 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:44,652 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:44,653 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:44,653 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:45,063 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:10:45,121 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:45,121 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:45,121 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:45,122 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:45,631 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:45,632 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:45,632 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:45,634 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:46,063 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:10:46,151 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:46,152 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:46,152 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:46,153 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:46,700 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:46,701 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:46,701 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:46,701 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:10:46,702 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:47,064 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:10:47,182 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:47,183 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:47,183 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:47,184 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:47,656 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:47,657 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:47,657 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:47,659 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:48,065 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:10:48,206 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:48,207 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:48,207 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:48,208 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:48,676 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:48,677 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:48,677 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:48,678 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:49,066 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:10:49,129 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:49,130 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:49,130 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:49,131 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:49,579 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:49,580 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:49,580 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:49,581 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:50,067 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:10:50,138 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:50,138 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:50,139 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:50,140 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:50,686 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:50,687 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:50,687 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:50,688 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:51,068 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:10:51,112 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:51,112 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:51,113 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:51,114 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:51,581 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:51,581 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:51,582 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:51,582 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:52,069 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:10:52,089 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 09:10:52,089 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: stop_status -2023-10-23 09:10:52,093 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:52,349 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:52,349 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:10:52,350 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:52,352 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:52,532 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:52,532 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:52,532 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:52,533 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:53,007 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:53,007 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:53,007 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:53,008 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:53,069 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:10:53,555 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:53,556 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:53,556 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:53,557 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:54,001 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:54,001 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:54,002 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:54,002 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:54,070 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:10:54,511 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:54,511 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:54,511 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:54,513 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:55,015 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:55,015 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:55,016 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:55,016 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:55,071 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:10:55,457 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:55,458 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:55,458 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:55,459 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:55,937 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:55,938 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:55,938 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:55,939 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:56,072 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:10:56,406 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:56,407 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:56,407 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:56,408 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:56,859 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:56,860 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:56,860 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:56,862 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:57,073 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:10:57,356 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:57,357 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:57,357 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:57,357 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:10:57,358 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:57,910 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:57,910 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:57,911 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:57,911 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:58,074 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:10:58,390 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:58,391 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:58,391 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:58,392 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:58,848 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:58,848 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:58,849 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:58,849 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:59,074 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:10:59,420 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:59,420 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:59,421 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:59,422 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:10:59,871 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:10:59,872 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:10:59,872 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:10:59,872 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:00,075 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:00,395 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:00,396 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:00,396 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:00,397 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:00,900 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:00,900 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:00,901 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:00,901 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:01,076 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:01,379 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:01,379 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:01,379 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:01,380 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:01,838 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:01,838 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:01,839 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:01,840 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:02,076 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:02,317 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:02,318 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:02,318 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:02,319 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:02,825 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:02,826 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:02,829 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:11:03,034 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:03,034 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:03,077 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/config.yaml -2023-10-23 09:11:03,077 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:03,271 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:03,271 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:03,271 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:03,272 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:03,742 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:03,743 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:03,743 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:03,744 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:04,078 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:04,286 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:04,286 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:04,286 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:04,288 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:04,767 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:04,767 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:04,767 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:04,768 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:05,079 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:05,227 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:05,228 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:05,228 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:05,229 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:05,692 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:05,693 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:05,693 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:05,694 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:06,079 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:06,114 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:06,115 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:06,115 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:06,115 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:06,732 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:06,733 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:06,733 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:06,735 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:07,080 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:07,278 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 09:11:07,279 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: stop_status -2023-10-23 09:11:07,282 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:07,542 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:07,542 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:07,543 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:07,775 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:07,776 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:07,776 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:07,777 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:08,081 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:08,257 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:08,257 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:08,257 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:08,258 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:11:08,258 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:08,886 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:08,887 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:08,887 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:08,889 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:09,082 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:09,348 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:09,349 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:09,349 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:09,349 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:09,851 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:09,851 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:09,851 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:09,852 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:10,082 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/output.log -2023-10-23 09:11:10,083 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:10,319 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:10,320 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:10,320 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:10,321 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:10,819 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:10,819 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:10,819 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:10,820 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:11,083 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:11,292 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:11,293 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:11,293 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:11,295 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:11,762 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:11,762 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:11,763 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:11,763 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:12,084 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:12,244 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:12,244 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:12,244 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:12,245 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:12,727 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:12,728 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:12,728 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:12,729 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:13,085 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:13,204 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:13,204 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:13,205 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:13,211 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:13,762 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:13,763 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:13,763 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:13,764 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:11:13,765 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:14,085 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:14,249 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:14,250 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:14,250 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:14,251 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:14,757 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:14,758 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:14,758 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:14,759 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:15,086 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:15,207 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:15,208 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:15,208 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:15,209 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:15,653 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:15,654 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:15,654 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:15,655 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:16,087 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:16,163 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:16,164 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:16,164 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:16,166 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:16,632 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:16,632 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:16,632 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:16,633 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:17,088 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:17,105 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:17,105 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:17,106 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:17,106 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:17,650 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:17,651 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:17,651 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:17,651 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:18,090 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:18,137 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:18,138 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:18,138 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:18,139 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:18,650 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:18,651 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:18,651 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:18,653 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:19,092 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:19,127 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:19,128 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:19,128 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:19,128 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:11:19,129 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:19,578 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:19,579 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:19,579 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:19,580 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:20,093 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:20,176 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:20,177 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:20,177 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:20,178 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:20,608 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:20,608 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:20,608 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:20,611 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:21,093 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:21,169 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:21,170 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:21,170 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:21,172 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:21,748 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:21,749 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:21,749 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:21,749 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:22,094 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:22,295 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 09:11:22,295 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: stop_status -2023-10-23 09:11:22,298 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:22,562 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:22,562 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:22,563 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:22,769 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:22,770 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:22,770 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:22,771 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:23,095 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:23,289 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:23,289 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:23,289 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:23,291 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:23,846 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:23,847 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:23,847 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:23,848 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:24,096 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:24,310 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:24,311 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:24,311 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:24,311 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:11:24,312 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:24,832 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:24,833 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:24,833 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:24,834 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:25,096 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:25,284 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:25,285 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:25,285 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:25,286 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:25,790 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:25,790 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:25,790 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:25,792 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:26,097 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:26,280 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:26,280 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:26,280 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:26,281 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:26,747 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:26,748 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:26,748 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:26,748 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:27,098 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:27,260 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:27,261 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:27,261 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:27,262 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:27,733 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:27,733 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:27,734 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:27,734 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:28,098 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:28,180 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:28,181 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:28,181 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:28,183 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:28,641 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:28,642 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:28,642 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:28,643 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:29,028 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:29,029 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:29,029 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:29,029 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:29,099 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:29,495 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:29,496 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:29,496 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:29,496 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:11:29,497 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:30,045 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:30,046 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:30,046 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:30,047 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:30,100 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:30,508 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:30,509 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:30,509 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:30,510 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:30,990 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:30,991 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:30,991 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:30,992 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:31,101 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:31,371 DEBUG SystemMonitor:54598 [system_monitor.py:_start():159] Starting system metrics aggregation loop -2023-10-23 09:11:31,372 DEBUG SenderThread:54598 [sender.py:send():375] send: stats -2023-10-23 09:11:31,540 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:31,541 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:31,541 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:31,542 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:32,060 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:32,061 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:32,061 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:32,062 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:32,102 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:32,502 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:32,502 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:32,503 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:32,503 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:32,998 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:32,998 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:32,998 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:32,999 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:33,102 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:33,468 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:33,468 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:33,468 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:33,469 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:33,994 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:33,995 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:33,995 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:33,996 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:34,103 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/output.log -2023-10-23 09:11:34,103 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:34,465 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:34,465 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:34,465 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:34,466 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:35,003 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:35,004 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:35,004 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:35,004 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:11:35,005 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:35,104 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:35,469 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:35,470 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:35,470 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:35,471 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:35,929 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:35,930 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:35,930 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:35,931 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:36,105 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:36,383 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:36,383 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:36,383 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:36,384 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:36,899 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:36,899 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:36,899 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:36,900 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:37,105 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:37,294 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 09:11:37,294 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: stop_status -2023-10-23 09:11:37,495 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:37,584 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:37,585 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:37,585 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:37,873 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:37,874 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:37,874 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:37,875 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:38,109 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:38,402 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:38,403 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:38,403 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:38,405 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:38,928 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:38,929 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:38,929 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:38,930 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:39,110 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:39,516 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:39,516 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:39,517 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:39,517 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:39,995 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:39,996 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:39,996 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:39,996 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:40,110 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:40,447 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:40,448 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:40,448 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:40,448 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:11:40,449 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:41,075 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:41,076 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:41,076 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:41,077 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:41,111 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:41,528 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:41,529 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:41,529 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:41,529 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:42,011 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:42,012 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:42,012 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:42,013 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:42,112 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:42,564 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:42,565 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:42,565 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:42,566 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:43,031 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:43,032 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:43,032 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:43,033 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:43,112 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:43,488 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:43,489 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:43,489 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:43,490 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:43,935 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:43,935 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:43,935 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:43,936 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:44,113 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:44,437 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:44,437 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:44,438 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:44,438 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:44,889 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:44,889 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:44,889 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:44,890 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:45,114 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:45,355 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:45,356 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:45,356 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:45,356 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:45,816 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:45,816 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:45,817 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:45,817 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:11:45,817 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:46,115 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:46,301 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:46,301 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:46,302 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:46,302 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:46,773 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:46,774 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:46,774 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:46,775 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:47,116 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:47,298 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:47,299 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:47,299 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:47,300 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:47,772 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:47,772 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:47,773 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:47,773 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:48,116 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:48,217 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:48,218 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:48,218 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:48,219 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:48,690 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:48,691 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:48,691 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:48,693 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:49,117 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:49,135 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:49,136 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:49,136 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:49,136 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:49,674 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:49,675 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:49,675 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:49,676 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:50,118 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:50,118 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:50,119 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:50,119 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:50,120 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:50,630 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:50,631 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:50,631 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:50,631 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:51,119 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:51,156 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:51,156 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:51,156 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:51,157 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:11:51,157 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:51,623 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:51,624 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:51,624 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:51,625 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:52,054 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:52,054 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:52,055 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:52,055 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:52,121 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:52,294 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 09:11:52,294 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: stop_status -2023-10-23 09:11:52,598 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:52,599 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:52,599 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:52,600 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:53,122 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:53,166 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:53,166 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:53,166 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:53,167 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:53,687 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:53,687 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:53,688 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:53,688 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:54,122 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:54,143 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:54,144 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:54,144 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:54,145 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:54,601 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:54,601 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:54,602 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:54,602 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:55,032 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:55,032 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:55,032 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:55,035 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:55,123 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:55,591 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:55,592 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:55,592 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:55,593 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:56,049 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:56,049 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:56,050 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:56,050 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:56,124 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:56,553 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:56,554 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:56,554 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:56,554 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:11:56,555 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:57,051 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:57,052 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:57,052 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:57,053 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:57,125 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:57,478 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:57,478 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:57,479 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:57,479 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:57,909 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:57,910 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:57,910 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:57,911 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:58,126 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/output.log -2023-10-23 09:11:58,126 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:58,318 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:58,318 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:58,318 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:58,319 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:58,869 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:58,869 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:58,869 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:58,871 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:59,126 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:11:59,339 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:59,340 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:59,340 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:59,341 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:11:59,788 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:11:59,789 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:11:59,789 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:11:59,790 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:00,127 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:00,237 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:00,238 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:00,238 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:00,239 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:00,687 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:00,687 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:00,687 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:00,688 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:01,128 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:01,219 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:01,219 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:01,219 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:01,220 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:01,373 DEBUG SenderThread:54598 [sender.py:send():375] send: stats -2023-10-23 09:12:01,667 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:01,668 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:01,668 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:01,668 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:12:01,669 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:02,129 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:02,197 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:02,198 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:02,198 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:02,199 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:02,727 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:02,727 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:02,727 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:02,728 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:03,129 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:03,154 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:03,154 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:03,155 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:03,155 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:03,624 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:03,625 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:03,625 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:03,626 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:04,130 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:04,155 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:04,155 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:04,155 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:04,156 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:04,742 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:04,743 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:04,743 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:04,744 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:05,131 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:05,197 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:05,197 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:05,197 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:05,198 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:05,680 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:05,681 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:05,681 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:05,682 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:06,132 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:06,166 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:06,167 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:06,167 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:06,168 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:06,674 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:06,675 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:06,675 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:06,675 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:12:06,676 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:07,123 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:07,124 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:07,124 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:07,124 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:07,132 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:07,294 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 09:12:07,294 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: stop_status -2023-10-23 09:12:07,594 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:07,595 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:07,595 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:07,596 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:08,061 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:08,062 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:08,062 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:08,062 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:08,133 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:08,489 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:08,490 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:08,490 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:08,491 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:08,940 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:08,941 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:08,941 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:08,943 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:09,134 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:09,413 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:09,414 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:09,414 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:09,415 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:09,894 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:09,895 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:09,895 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:09,896 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:10,135 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:10,372 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:10,372 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:10,372 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:10,373 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:10,880 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:10,880 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:10,880 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:10,881 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:11,135 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:11,394 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:11,395 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:11,395 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:11,396 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:11,949 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:11,950 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:11,950 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:11,950 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:12:11,951 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:12,136 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:12,472 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:12,472 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:12,473 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:12,473 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:12,992 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:12,993 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:12,993 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:12,994 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:13,137 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:13,524 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:13,525 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:13,525 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:13,526 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:14,029 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:14,030 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:14,030 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:14,031 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:14,138 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:14,592 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:14,593 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:14,593 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:14,594 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:15,091 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:15,092 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:15,092 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:15,093 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:15,139 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:15,639 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:15,640 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:15,640 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:15,641 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:16,118 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:16,118 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:16,119 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:16,119 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:16,139 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:16,547 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:16,547 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:16,547 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:16,548 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:17,105 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:17,106 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:17,106 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:17,106 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:12:17,107 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:17,140 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:17,577 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:17,577 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:17,577 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:17,578 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:18,141 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:18,170 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:18,171 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:18,171 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:18,172 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:18,667 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:18,667 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:18,668 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:18,668 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:19,094 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:19,094 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:19,094 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:19,096 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:19,142 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:19,692 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:19,693 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:19,693 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:19,694 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:20,142 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:20,228 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:20,229 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:20,229 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:20,230 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:20,672 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:20,672 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:20,672 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:20,673 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:21,143 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:21,156 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:21,157 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:21,157 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:21,158 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:21,718 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:21,718 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:21,718 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:21,719 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:22,144 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:22,187 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:22,187 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:22,187 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:22,187 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:12:22,188 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:22,298 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 09:12:22,298 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: stop_status -2023-10-23 09:12:22,786 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:22,787 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:22,787 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:22,788 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:23,145 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:23,324 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:23,325 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:23,325 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:23,326 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:23,809 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:23,810 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:23,810 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:23,811 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:24,145 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/output.log -2023-10-23 09:12:24,145 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:24,276 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:24,277 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:24,277 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:24,278 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:24,797 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:24,797 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:24,797 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:24,798 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:25,146 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:25,336 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:25,337 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:25,337 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:25,337 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:25,813 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:25,814 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:25,814 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:25,814 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:26,147 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:26,328 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:26,329 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:26,329 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:26,330 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:26,770 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:26,771 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:26,771 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:26,772 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:27,148 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:27,263 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:27,264 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:27,264 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:27,264 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:12:27,265 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:27,653 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:27,653 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:27,653 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:27,654 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:28,148 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:28,219 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:28,220 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:28,220 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:28,221 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:28,685 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:28,686 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:28,686 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:28,687 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:29,145 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:29,146 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:29,146 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:29,149 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:29,149 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:29,703 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:29,704 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:29,704 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:29,705 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:30,149 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:30,184 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:30,184 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:30,184 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:30,185 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:30,729 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:30,730 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:30,730 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:30,731 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:31,150 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:31,236 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:31,237 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:31,237 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:31,240 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:31,374 DEBUG SenderThread:54598 [sender.py:send():375] send: stats -2023-10-23 09:12:31,738 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:31,738 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:31,739 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:31,739 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:32,151 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:32,248 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:32,248 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:32,248 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:32,249 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:32,654 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:32,655 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:32,655 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:32,655 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:12:32,656 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:33,108 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:33,108 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:33,109 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:33,111 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:33,152 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:33,666 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:33,667 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:33,667 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:33,668 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:34,152 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:34,210 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:34,211 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:34,211 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:34,212 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:34,654 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:34,654 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:34,655 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:34,655 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:35,101 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:35,102 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:35,102 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:35,102 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:35,153 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:35,651 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:35,651 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:35,652 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:35,652 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:36,098 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:36,098 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:36,098 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:36,099 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:36,154 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:36,644 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:36,645 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:36,645 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:36,646 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:37,153 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:37,154 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:37,154 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:37,155 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:37,155 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:37,296 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: stop_status -2023-10-23 09:12:37,296 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: stop_status -2023-10-23 09:12:37,583 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:37,583 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:37,583 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:37,584 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:38,155 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:38,180 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:38,180 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:38,181 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:38,181 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:12:38,181 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:38,727 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:38,728 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:38,728 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:38,729 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:39,156 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:39,226 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:39,227 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:39,227 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:39,229 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:39,735 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:39,736 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:39,736 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:39,737 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:40,157 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:40,216 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:40,216 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:40,216 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:40,219 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:40,690 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:40,690 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:40,691 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:40,691 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:41,158 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:41,261 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:41,261 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:41,261 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:41,262 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:41,715 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:41,716 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:41,716 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:41,716 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:42,159 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:42,285 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:42,286 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:42,286 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:42,287 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:42,766 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:42,767 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:42,767 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:42,768 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:43,159 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:43,212 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:43,213 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:43,213 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:43,213 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:12:43,214 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:43,666 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:43,666 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:43,667 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:43,667 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:44,163 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:44,226 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:44,227 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:44,227 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:44,228 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:44,732 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:44,732 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:44,732 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:44,733 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:45,159 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:45,159 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:45,160 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:45,161 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:45,163 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:45,663 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:45,664 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:45,664 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:45,665 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:46,115 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:46,115 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:46,116 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:46,117 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:46,164 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:46,562 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:46,563 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:46,563 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:46,564 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:47,113 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:47,113 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:47,113 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:47,117 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:47,165 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:47,557 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:47,557 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:47,558 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:47,558 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:48,122 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:48,122 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:48,122 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:48,123 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:48,166 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/output.log -2023-10-23 09:12:48,166 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:48,601 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:48,601 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:48,602 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:48,602 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:12:48,602 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:49,082 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:49,083 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:49,083 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:49,084 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:49,167 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:49,646 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:49,647 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:49,647 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:49,649 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:50,167 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:50,179 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: partial_history -2023-10-23 09:12:50,180 DEBUG SenderThread:54598 [sender.py:send():375] send: history -2023-10-23 09:12:50,180 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: summary_record -2023-10-23 09:12:50,181 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:50,543 DEBUG SenderThread:54598 [sender.py:send():375] send: exit -2023-10-23 09:12:50,543 INFO SenderThread:54598 [sender.py:send_exit():598] handling exit code: 255 -2023-10-23 09:12:50,543 INFO SenderThread:54598 [sender.py:send_exit():600] handling runtime: 139 -2023-10-23 09:12:50,544 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:50,544 INFO SenderThread:54598 [sender.py:send_exit():606] send defer -2023-10-23 09:12:50,544 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:12:50,544 INFO HandlerThread:54598 [handler.py:handle_request_defer():170] handle defer: 0 -2023-10-23 09:12:50,545 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: defer -2023-10-23 09:12:50,545 INFO SenderThread:54598 [sender.py:send_request_defer():622] handle sender defer: 0 -2023-10-23 09:12:50,545 INFO SenderThread:54598 [sender.py:transition_state():626] send defer: 1 -2023-10-23 09:12:50,545 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:12:50,545 INFO HandlerThread:54598 [handler.py:handle_request_defer():170] handle defer: 1 -2023-10-23 09:12:50,545 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: defer -2023-10-23 09:12:50,545 INFO SenderThread:54598 [sender.py:send_request_defer():622] handle sender defer: 1 -2023-10-23 09:12:50,545 INFO SenderThread:54598 [sender.py:transition_state():626] send defer: 2 -2023-10-23 09:12:50,545 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:12:50,545 INFO HandlerThread:54598 [handler.py:handle_request_defer():170] handle defer: 2 -2023-10-23 09:12:50,545 INFO HandlerThread:54598 [system_monitor.py:finish():190] Stopping system monitor -2023-10-23 09:12:50,545 DEBUG SystemMonitor:54598 [system_monitor.py:_start():166] Finished system metrics aggregation loop -2023-10-23 09:12:50,545 DEBUG SystemMonitor:54598 [system_monitor.py:_start():170] Publishing last batch of metrics -2023-10-23 09:12:50,546 INFO HandlerThread:54598 [interfaces.py:finish():202] Joined cpu monitor -2023-10-23 09:12:50,547 INFO HandlerThread:54598 [interfaces.py:finish():202] Joined disk monitor -2023-10-23 09:12:50,658 INFO HandlerThread:54598 [interfaces.py:finish():202] Joined gpu monitor -2023-10-23 09:12:50,658 INFO HandlerThread:54598 [interfaces.py:finish():202] Joined memory monitor -2023-10-23 09:12:50,658 INFO HandlerThread:54598 [interfaces.py:finish():202] Joined network monitor -2023-10-23 09:12:50,659 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: defer -2023-10-23 09:12:50,659 INFO SenderThread:54598 [sender.py:send_request_defer():622] handle sender defer: 2 -2023-10-23 09:12:50,659 INFO SenderThread:54598 [sender.py:transition_state():626] send defer: 3 -2023-10-23 09:12:50,659 DEBUG SenderThread:54598 [sender.py:send():375] send: stats -2023-10-23 09:12:50,659 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:12:50,660 INFO HandlerThread:54598 [handler.py:handle_request_defer():170] handle defer: 3 -2023-10-23 09:12:50,660 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: defer -2023-10-23 09:12:50,660 INFO SenderThread:54598 [sender.py:send_request_defer():622] handle sender defer: 3 -2023-10-23 09:12:50,660 INFO SenderThread:54598 [sender.py:transition_state():626] send defer: 4 -2023-10-23 09:12:50,660 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:12:50,660 INFO HandlerThread:54598 [handler.py:handle_request_defer():170] handle defer: 4 -2023-10-23 09:12:50,660 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: defer -2023-10-23 09:12:50,660 INFO SenderThread:54598 [sender.py:send_request_defer():622] handle sender defer: 4 -2023-10-23 09:12:50,660 INFO SenderThread:54598 [sender.py:transition_state():626] send defer: 5 -2023-10-23 09:12:50,660 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:12:50,660 INFO HandlerThread:54598 [handler.py:handle_request_defer():170] handle defer: 5 -2023-10-23 09:12:50,660 DEBUG SenderThread:54598 [sender.py:send():375] send: summary -2023-10-23 09:12:50,661 INFO SenderThread:54598 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end -2023-10-23 09:12:50,661 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: defer -2023-10-23 09:12:50,661 INFO SenderThread:54598 [sender.py:send_request_defer():622] handle sender defer: 5 -2023-10-23 09:12:50,661 INFO SenderThread:54598 [sender.py:transition_state():626] send defer: 6 -2023-10-23 09:12:50,661 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:12:50,661 INFO HandlerThread:54598 [handler.py:handle_request_defer():170] handle defer: 6 -2023-10-23 09:12:50,661 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: defer -2023-10-23 09:12:50,661 INFO SenderThread:54598 [sender.py:send_request_defer():622] handle sender defer: 6 -2023-10-23 09:12:50,662 INFO SenderThread:54598 [sender.py:transition_state():626] send defer: 7 -2023-10-23 09:12:50,662 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: status_report -2023-10-23 09:12:50,662 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:12:50,662 INFO HandlerThread:54598 [handler.py:handle_request_defer():170] handle defer: 7 -2023-10-23 09:12:50,662 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: defer -2023-10-23 09:12:50,662 INFO SenderThread:54598 [sender.py:send_request_defer():622] handle sender defer: 7 -2023-10-23 09:12:51,168 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:51,543 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: poll_exit -2023-10-23 09:12:52,169 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/output.log -2023-10-23 09:12:53,441 INFO SenderThread:54598 [sender.py:transition_state():626] send defer: 8 -2023-10-23 09:12:53,441 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: poll_exit -2023-10-23 09:12:53,441 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:12:53,441 INFO HandlerThread:54598 [handler.py:handle_request_defer():170] handle defer: 8 -2023-10-23 09:12:53,442 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: defer -2023-10-23 09:12:53,442 INFO SenderThread:54598 [sender.py:send_request_defer():622] handle sender defer: 8 -2023-10-23 09:12:53,450 INFO SenderThread:54598 [sender.py:transition_state():626] send defer: 9 -2023-10-23 09:12:53,450 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:12:53,450 DEBUG SenderThread:54598 [sender.py:send():375] send: artifact -2023-10-23 09:12:53,450 INFO HandlerThread:54598 [handler.py:handle_request_defer():170] handle defer: 9 -2023-10-23 09:12:54,170 INFO Thread-12 :54598 [dir_watcher.py:_on_file_modified():295] file/dir modified: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/output.log -2023-10-23 09:12:54,483 INFO SenderThread:54598 [sender.py:send_artifact():1474] sent artifact job-https___github.com_junchen14_MiniGPT-4_finetune_train.py - {'id': 'QXJ0aWZhY3Q6NjEzMjczODU5', 'digest': 'e1357229d7190882bf8b2dba86a37a56', 'state': 'COMMITTED', 'aliases': [{'artifactCollectionName': 'job-https___github.com_junchen14_MiniGPT-4_finetune_train.py', 'alias': 'latest'}, {'artifactCollectionName': 'job-https___github.com_junchen14_MiniGPT-4_finetune_train.py', 'alias': 'v0'}], 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjEwOTExOTI4MQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6NjEzMjczODU5', 'versionIndex': 0}}, 'version': 'v0'} -2023-10-23 09:12:54,483 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: defer -2023-10-23 09:12:54,483 INFO SenderThread:54598 [sender.py:send_request_defer():622] handle sender defer: 9 -2023-10-23 09:12:54,484 INFO SenderThread:54598 [dir_watcher.py:finish():365] shutting down directory watcher -2023-10-23 09:12:55,171 INFO SenderThread:54598 [dir_watcher.py:finish():395] scan: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files -2023-10-23 09:12:55,172 INFO SenderThread:54598 [dir_watcher.py:finish():409] scan save: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/config.yaml config.yaml -2023-10-23 09:12:55,172 INFO SenderThread:54598 [dir_watcher.py:finish():409] scan save: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/conda-environment.yaml conda-environment.yaml -2023-10-23 09:12:55,172 INFO SenderThread:54598 [dir_watcher.py:finish():409] scan save: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json wandb-summary.json -2023-10-23 09:12:55,177 INFO SenderThread:54598 [dir_watcher.py:finish():409] scan save: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/output.log output.log -2023-10-23 09:12:55,177 INFO SenderThread:54598 [dir_watcher.py:finish():409] scan save: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/requirements.txt requirements.txt -2023-10-23 09:12:55,180 INFO SenderThread:54598 [dir_watcher.py:finish():409] scan save: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-metadata.json wandb-metadata.json -2023-10-23 09:12:55,180 INFO SenderThread:54598 [sender.py:transition_state():626] send defer: 10 -2023-10-23 09:12:55,182 DEBUG HandlerThread:54598 [handler.py:handle_request():144] handle_request: defer -2023-10-23 09:12:55,182 INFO HandlerThread:54598 [handler.py:handle_request_defer():170] handle defer: 10 -2023-10-23 09:12:55,185 DEBUG SenderThread:54598 [sender.py:send_request():402] send_request: defer -2023-10-23 09:12:55,185 WARNING StreamThr :54598 [internal.py:is_dead():414] Internal process exiting, parent pid 53639 disappeared -2023-10-23 09:12:55,185 INFO SenderThread:54598 [sender.py:send_request_defer():622] handle sender defer: 10 -2023-10-23 09:12:55,185 ERROR StreamThr :54598 [internal.py:wandb_internal():152] Internal process shutdown. -2023-10-23 09:12:55,185 INFO SenderThread:54598 [file_pusher.py:finish():168] shutting down file pusher -2023-10-23 09:12:55,185 INFO SenderThread:54598 [sender.py:finish():1550] shutting down sender -2023-10-23 09:12:55,186 INFO SenderThread:54598 [file_pusher.py:finish():168] shutting down file pusher -2023-10-23 09:12:55,186 INFO SenderThread:54598 [file_pusher.py:join():173] waiting for file pusher -2023-10-23 09:12:55,753 INFO wandb-upload_1:54598 [upload_job.py:push():137] Uploaded file /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/conda-environment.yaml -2023-10-23 09:12:55,986 INFO wandb-upload_0:54598 [upload_job.py:push():137] Uploaded file /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/config.yaml -2023-10-23 09:12:56,124 INFO wandb-upload_2:54598 [upload_job.py:push():137] Uploaded file /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/wandb-summary.json -2023-10-23 09:12:56,182 INFO HandlerThread:54598 [handler.py:finish():845] shutting down handler -2023-10-23 09:12:56,183 INFO WriterThread:54598 [datastore.py:close():298] close: /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/run-25yuqsbo.wandb -2023-10-23 09:12:56,210 INFO wandb-upload_4:54598 [upload_job.py:push():137] Uploaded file /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/requirements.txt -2023-10-23 09:12:56,248 INFO wandb-upload_3:54598 [upload_job.py:push():137] Uploaded file /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/files/output.log -2023-10-23 09:12:56,448 INFO Thread-11 :54598 [sender.py:transition_state():626] send defer: 11 diff --git a/wandb/run-20231023_091030-25yuqsbo/logs/debug.log b/wandb/run-20231023_091030-25yuqsbo/logs/debug.log deleted file mode 100644 index e1f2ca1..0000000 --- a/wandb/run-20231023_091030-25yuqsbo/logs/debug.log +++ /dev/null @@ -1,26 +0,0 @@ -2023-10-23 09:10:30,032 INFO MainThread:53639 [wandb_setup.py:_flush():76] Configure stats pid to 53639 -2023-10-23 09:10:30,032 INFO MainThread:53639 [wandb_setup.py:_flush():76] Loading settings from /home/chenj0g/.config/wandb/settings -2023-10-23 09:10:30,032 INFO MainThread:53639 [wandb_setup.py:_flush():76] Loading settings from /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/settings -2023-10-23 09:10:30,032 INFO MainThread:53639 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} -2023-10-23 09:10:30,032 INFO MainThread:53639 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'train.py', 'program': '/ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/train.py'} -2023-10-23 09:10:30,033 INFO MainThread:53639 [wandb_init.py:_log_setup():507] Logging user logs to /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/logs/debug.log -2023-10-23 09:10:30,033 INFO MainThread:53639 [wandb_init.py:_log_setup():508] Logging internal logs to /ibex/project/c2133/minigpt2_finetune/MiniGPT-4_finetune/wandb/run-20231023_091030-25yuqsbo/logs/debug-internal.log -2023-10-23 09:10:30,033 INFO MainThread:53639 [wandb_init.py:init():547] calling init triggers -2023-10-23 09:10:30,033 INFO MainThread:53639 [wandb_init.py:init():554] wandb.init called with sweep_config: {} -config: {} -2023-10-23 09:10:30,033 INFO MainThread:53639 [wandb_init.py:init():595] starting backend -2023-10-23 09:10:30,033 INFO MainThread:53639 [wandb_init.py:init():599] setting up manager -2023-10-23 09:10:30,034 INFO MainThread:53639 [backend.py:_multiprocessing_setup():106] multiprocessing start_methods=fork,spawn,forkserver, using: spawn -2023-10-23 09:10:30,036 INFO MainThread:53639 [wandb_init.py:init():605] backend started and connected -2023-10-23 09:10:30,038 INFO MainThread:53639 [wandb_init.py:init():695] updated telemetry -2023-10-23 09:10:30,456 INFO MainThread:53639 [wandb_init.py:init():732] communicating run to backend with 60.0 second timeout -2023-10-23 09:10:31,052 INFO MainThread:53639 [wandb_run.py:_on_init():2176] communicating current version -2023-10-23 09:10:31,359 INFO MainThread:53639 [wandb_run.py:_on_init():2185] got version response upgrade_message: "wandb version 0.15.12 is available! To upgrade, please run:\n $ pip install wandb --upgrade" - -2023-10-23 09:10:31,359 INFO MainThread:53639 [wandb_init.py:init():782] starting run threads in backend -2023-10-23 09:10:36,972 INFO MainThread:53639 [wandb_run.py:_console_start():2157] atexit reg -2023-10-23 09:10:36,972 INFO MainThread:53639 [wandb_run.py:_redirect():2012] redirect: SettingsConsole.WRAP_RAW -2023-10-23 09:10:36,972 INFO MainThread:53639 [wandb_run.py:_redirect():2077] Wrapping output streams. -2023-10-23 09:10:36,972 INFO MainThread:53639 [wandb_run.py:_redirect():2102] Redirects installed. -2023-10-23 09:10:36,972 INFO MainThread:53639 [wandb_init.py:init():824] run started, returning control to user process -2023-10-23 09:10:36,973 INFO MainThread:53639 [wandb_watch.py:watch():51] Watching diff --git a/wandb/run-20231023_091030-25yuqsbo/run-25yuqsbo.wandb b/wandb/run-20231023_091030-25yuqsbo/run-25yuqsbo.wandb deleted file mode 100644 index ef667cb..0000000 Binary files a/wandb/run-20231023_091030-25yuqsbo/run-25yuqsbo.wandb and /dev/null differ