diff --git a/004_image_stack_ram_based_reward_custom/__pycache__/street_fighter_custom_wrapper.cpython-38.pyc b/004_image_stack_ram_based_reward_custom/__pycache__/street_fighter_custom_wrapper.cpython-38.pyc index d473d0a..859eef5 100644 Binary files a/004_image_stack_ram_based_reward_custom/__pycache__/street_fighter_custom_wrapper.cpython-38.pyc and b/004_image_stack_ram_based_reward_custom/__pycache__/street_fighter_custom_wrapper.cpython-38.pyc differ diff --git a/004_image_stack_ram_based_reward_custom/check_reward.py b/004_image_stack_ram_based_reward_custom/check_reward.py index 0821924..b3b7256 100644 --- a/004_image_stack_ram_based_reward_custom/check_reward.py +++ b/004_image_stack_ram_based_reward_custom/check_reward.py @@ -1,3 +1,4 @@ +import os import time import retro @@ -5,6 +6,9 @@ from stable_baselines3.common.monitor import Monitor from street_fighter_custom_wrapper import StreetFighterCustomWrapper +LOG_DIR = 'logs/' +os.makedirs(LOG_DIR, exist_ok=True) + def make_env(game, state): def _init(): env = retro.make( diff --git a/004_image_stack_ram_based_reward_custom/logs/PPO_10/events.out.tfevents.1680505905.DESKTOP-9E17TO7.28664.0 b/004_image_stack_ram_based_reward_custom/logs/PPO_10/events.out.tfevents.1680505905.DESKTOP-9E17TO7.28664.0 new file mode 100644 index 0000000..4091b1f Binary files /dev/null and b/004_image_stack_ram_based_reward_custom/logs/PPO_10/events.out.tfevents.1680505905.DESKTOP-9E17TO7.28664.0 differ diff --git a/004_image_stack_ram_based_reward_custom/logs/PPO_11/events.out.tfevents.1680509256.DESKTOP-9E17TO7.5124.0 b/004_image_stack_ram_based_reward_custom/logs/PPO_11/events.out.tfevents.1680509256.DESKTOP-9E17TO7.5124.0 new file mode 100644 index 0000000..8784d0d Binary files /dev/null and b/004_image_stack_ram_based_reward_custom/logs/PPO_11/events.out.tfevents.1680509256.DESKTOP-9E17TO7.5124.0 differ diff --git a/004_image_stack_ram_based_reward_custom/logs/PPO_12/events.out.tfevents.1680515264.DESKTOP-9E17TO7.29580.0 b/004_image_stack_ram_based_reward_custom/logs/PPO_12/events.out.tfevents.1680515264.DESKTOP-9E17TO7.29580.0 new file mode 100644 index 0000000..beda370 Binary files /dev/null and b/004_image_stack_ram_based_reward_custom/logs/PPO_12/events.out.tfevents.1680515264.DESKTOP-9E17TO7.29580.0 differ diff --git a/004_image_stack_ram_based_reward_custom/logs/PPO_3/events.out.tfevents.1680450538.DESKTOP-9E17TO7.4520.0 b/004_image_stack_ram_based_reward_custom/logs/PPO_3/events.out.tfevents.1680450538.DESKTOP-9E17TO7.4520.0 index 0d60db2..611dfee 100644 Binary files a/004_image_stack_ram_based_reward_custom/logs/PPO_3/events.out.tfevents.1680450538.DESKTOP-9E17TO7.4520.0 and b/004_image_stack_ram_based_reward_custom/logs/PPO_3/events.out.tfevents.1680450538.DESKTOP-9E17TO7.4520.0 differ diff --git a/004_image_stack_ram_based_reward_custom/logs/PPO_4/events.out.tfevents.1680494082.DESKTOP-9E17TO7.1808.0 b/004_image_stack_ram_based_reward_custom/logs/PPO_4/events.out.tfevents.1680494082.DESKTOP-9E17TO7.1808.0 new file mode 100644 index 0000000..d6c046e Binary files /dev/null and b/004_image_stack_ram_based_reward_custom/logs/PPO_4/events.out.tfevents.1680494082.DESKTOP-9E17TO7.1808.0 differ diff --git a/004_image_stack_ram_based_reward_custom/logs/PPO_5/events.out.tfevents.1680501562.DESKTOP-9E17TO7.12740.0 b/004_image_stack_ram_based_reward_custom/logs/PPO_5/events.out.tfevents.1680501562.DESKTOP-9E17TO7.12740.0 new file mode 100644 index 0000000..9d71876 Binary files /dev/null and b/004_image_stack_ram_based_reward_custom/logs/PPO_5/events.out.tfevents.1680501562.DESKTOP-9E17TO7.12740.0 differ diff --git a/004_image_stack_ram_based_reward_custom/logs/PPO_6/events.out.tfevents.1680502870.DESKTOP-9E17TO7.21800.0 b/004_image_stack_ram_based_reward_custom/logs/PPO_6/events.out.tfevents.1680502870.DESKTOP-9E17TO7.21800.0 new file mode 100644 index 0000000..3802dc5 Binary files /dev/null and b/004_image_stack_ram_based_reward_custom/logs/PPO_6/events.out.tfevents.1680502870.DESKTOP-9E17TO7.21800.0 differ diff --git a/004_image_stack_ram_based_reward_custom/logs/PPO_7/events.out.tfevents.1680503637.DESKTOP-9E17TO7.30508.0 b/004_image_stack_ram_based_reward_custom/logs/PPO_7/events.out.tfevents.1680503637.DESKTOP-9E17TO7.30508.0 new file mode 100644 index 0000000..d2680ff Binary files /dev/null and b/004_image_stack_ram_based_reward_custom/logs/PPO_7/events.out.tfevents.1680503637.DESKTOP-9E17TO7.30508.0 differ diff --git a/004_image_stack_ram_based_reward_custom/logs/PPO_8/events.out.tfevents.1680504617.DESKTOP-9E17TO7.5124.0 b/004_image_stack_ram_based_reward_custom/logs/PPO_8/events.out.tfevents.1680504617.DESKTOP-9E17TO7.5124.0 new file mode 100644 index 0000000..bf853a7 Binary files /dev/null and b/004_image_stack_ram_based_reward_custom/logs/PPO_8/events.out.tfevents.1680504617.DESKTOP-9E17TO7.5124.0 differ diff --git a/004_image_stack_ram_based_reward_custom/logs/PPO_9/events.out.tfevents.1680505010.DESKTOP-9E17TO7.28200.0 b/004_image_stack_ram_based_reward_custom/logs/PPO_9/events.out.tfevents.1680505010.DESKTOP-9E17TO7.28200.0 new file mode 100644 index 0000000..5663415 Binary files /dev/null and b/004_image_stack_ram_based_reward_custom/logs/PPO_9/events.out.tfevents.1680505010.DESKTOP-9E17TO7.28200.0 differ diff --git a/004_image_stack_ram_based_reward_custom/logs/monitor.csv b/004_image_stack_ram_based_reward_custom/logs/monitor.csv index 680323f..137bced 100644 --- a/004_image_stack_ram_based_reward_custom/logs/monitor.csv +++ b/004_image_stack_ram_based_reward_custom/logs/monitor.csv @@ -1,203 +1,2047 @@ -#{"t_start": 1680450538.0135336, "env_id": null} +#{"t_start": 1680515263.2477837, "env_id": null} r,l,t -140,2721,9.666247 -475,2051,16.014586 -659,1841,22.078956 -767,1374,25.788732 -253,2591,35.3779 -337,2369,42.159742 -280,2490,51.242315 -316,2761,59.177625 -498,1532,65.235702 -794,1294,68.963929 -758,1295,72.564837 -343,1862,78.748257 -366,1293,82.443477 -344,3356,95.206057 -655,1716,99.877595 -717,1496,105.722482 -410,2494,113.515226 -906,1496,119.400876 -230,1769,124.036038 -966,1043,127.542873 -321,2284,136.329181 -812,1519,140.423345 -519,1729,146.851307 -722,1412,150.934866 -700,1646,157.093265 -387,1965,163.827162 -343,2604,171.611118 -123,1625,177.426801 -417,3407,187.666056 -187,2616,196.882013 -26,2142,203.498234 -1056,1218,207.239627 -927,1124,210.857355 -47,1994,217.59863 -510,1774,223.863903 -377,1381,227.737895 -481,1390,233.374429 -658,1447,237.396851 -349,3437,248.157832 -718,1829,254.408287 -180,2025,261.068938 -313,1839,267.477898 -902,975,270.676313 -292,2950,280.273983 -182,2268,287.275446 -187,1953,293.603531 -362,3130,303.389702 -638,1779,309.646411 -228,2179,316.57769 -341,2109,323.229764 -85,2241,330.295613 -260,3871,343.38057 --72,2846,353.025726 -319,2520,360.176952 -121,2334,367.161462 --143,2696,376.380194 --10,2199,383.130368 -254,3800,395.603224 -51,3540,406.048309 -340,2647,415.170668 -331,6905,437.351264 -741,1698,441.462111 -331,3519,454.219213 -489,2276,461.541512 -368,3599,472.906958 --126,3151,482.855691 --235,3220,494.629744 --17,2824,502.354869 --82,3238,514.654498 --160,3254,525.209512 --150,2923,535.200898 --93,7337,558.950392 --275,2822,566.908611 -245,3758,579.955454 --117,2586,589.282403 --230,2788,597.257352 -90,2896,607.108898 -967,1681,613.189581 --162,3178,623.550789 --111,2603,632.824294 --111,2160,639.682543 --91,3059,649.811759 -552,3472,660.344764 -156,2905,669.981916 -506,1973,676.494263 --29,3483,687.2097 --342,2388,694.675364 --98,7522,720.510979 -268,3198,730.537025 -123,7866,755.786125 -41,3279,766.078602 -454,3852,778.612931 -241,3330,788.852483 --197,2695,796.350531 -16,2708,805.798072 --276,2921,815.975549 -230,7838,842.84947 --343,2955,852.604732 --124,2710,860.170467 --318,1477,866.041014 --37,3970,877.800582 --26,2989,888.043861 -732,8207,914.516506 -326,3080,924.643157 --189,2372,933.448068 -275,8785,961.099634 --189,2881,970.522553 -108,3190,980.352367 --351,2851,989.817974 --287,3287,1000.238004 --262,2847,1010.036809 --206,7848,1034.80143 -102,7532,1058.580046 -0,8037,1084.80434 -110,4073,1097.11781 -421,3413,1107.327183 --203,3154,1117.49272 -655,14205,1161.34724 -126,3993,1174.695066 -48,3832,1187.983629 -68,2995,1197.869093 -80,3252,1207.617986 -84,3776,1219.716212 --192,3176,1229.660275 --143,2819,1237.98842 -10,2730,1247.57504 --191,2460,1256.580759 --28,2546,1263.612297 -192,3534,1273.532951 -268,3797,1285.940749 --98,3139,1296.872381 -75,3568,1309.84493 --123,7274,1332.694059 -326,3440,1342.804182 -349,3737,1355.733267 -22,2943,1366.020118 --202,3018,1375.968116 -888,1928,1382.114744 --209,1646,1386.164465 -46,1613,1391.748134 --318,2434,1398.519993 --275,2288,1405.521706 -397,3578,1418.447945 -317,2150,1425.714679 --75,2716,1435.669649 --93,2679,1442.908609 -564,2987,1452.369631 -216,2904,1461.84269 -44,2300,1468.801676 -401,1470,1474.693752 -381,3590,1485.786877 -256,2522,1494.610591 --141,1773,1498.837729 -335,2651,1507.840033 -860,1561,1511.859426 -357,1743,1517.856122 -846,1433,1523.547 -702,720,1524.743536 -81,3314,1535.593991 -608,1468,1541.5119 -464,2507,1549.079192 -382,1465,1554.894497 -661,2153,1561.58434 --220,2172,1568.434792 -470,2597,1577.624233 -606,1471,1581.617123 -128,2485,1589.245833 --151,2076,1596.189308 --34,1775,1602.548944 -7,2518,1611.704006 --73,1256,1615.417475 -981,952,1618.628284 -537,1555,1622.779646 -336,2464,1631.451718 -490,2070,1638.048966 -337,3439,1648.578499 -367,2505,1657.904252 -365,2554,1665.472756 -654,1061,1669.007638 -334,3193,1679.491146 --125,1751,1685.74332 -342,2740,1695.388833 -541,1674,1699.803759 -303,3218,1709.898359 -62,2140,1716.650506 -37,1838,1722.973549 --9,2999,1732.744556 --47,2898,1742.188218 -462,3518,1752.401364 -206,2255,1760.763199 -494,2294,1767.674661 -198,2530,1775.444748 -149,2196,1782.305408 -593,2317,1791.355 -349,2208,1798.589246 --74,1673,1804.620136 -41,2712,1811.981201 -432,2759,1821.619351 -75,2880,1831.46073 -397,1858,1837.717627 --204,3008,1845.908291 +0.19523,1437,6.158065 +0.515301,2178,12.733591 +0.634529,1375,16.361347 +0.434956,1856,22.400417 +0.501522,1312,25.952795 +0.625168,1210,30.836148 +0.458572,2566,37.943796 +0.351876,2734,57.698138 +0.438391,2093,64.435742 +0.560551,1632,68.687624 +0.353,2766,77.604214 +0.385311,1908,83.546733 +0.615739,956,86.49515 +-0.126489,1632,92.323955 +0.344528,2451,99.244247 +0.546109,2006,105.640446 +0.205816,2114,112.294221 +0.361,2228,119.070559 +0.063959,1044,122.175435 +-0.131312,1145,125.693627 +0.53285,1694,131.350782 +0.213564,1301,134.872645 +-0.107708,1334,140.352893 +0.422398,1907,144.803949 +0.37097,2021,151.068017 +-0.125282,1212,156.016458 +0.553221,1058,159.176978 +0.428607,1882,163.966119 +0.166701,1974,170.56537 +0.095926,1761,176.806112 +0.332661,1719,182.924983 +0.235783,1861,189.180695 +0.188014,1399,192.797684 +0.184866,1113,195.983856 +0.182883,971,198.910249 +0.519682,1859,205.101854 +-0.092675,1103,208.582289 +0.012139,1399,212.352561 +0.252488,1761,218.047523 +0.193297,1857,224.25977 +0.208692,1949,230.906981 +0.333621,2097,237.512891 +0.337773,2379,244.75255 +0.2513,1880,251.02444 +0.118238,1465,254.738834 +0.143272,1599,260.233152 +0.115238,1035,263.322387 +0.222488,1903,269.358855 +0.151519,1236,272.746762 +0.330737,2823,281.849735 +0.147921,1557,286.12499 +-0.082913,986,289.469522 +0.172711,1425,295.147659 +0.200396,1914,300.300695 +0.115668,1525,306.026715 +0.328661,2302,312.809297 +0.439802,2048,319.077458 +0.113238,2099,325.299501 +0.474391,2093,331.803078 +0.26322,1875,337.829412 +0.31958,1782,344.011115 +0.304042,2600,351.556374 +0.459925,2169,358.451489 +0.3998,2410,367.434605 +0.181701,1662,371.541217 +0.491624,2159,379.708102 +0.397427,1538,383.485948 +0.31445,2858,392.260264 +0.547118,2070,398.407839 +0.511811,2341,405.493453 +0.549531,1588,411.42555 +0.205692,1546,415.700895 +0.5381,1899,422.417346 +0.251053,2578,431.415421 +0.612009,1386,435.04311 +0.43306,2288,441.672518 +0.508798,1455,446.981311 +0.259453,1552,450.913679 +0.04349,1699,456.589592 +0.524278,1131,460.000105 +0.277589,1934,466.343967 +0.007862,971,469.754009 +0.117344,1503,473.722137 +-0.219034,1166,477.240383 +0.015139,932,480.426492 +0.737164,939,483.609485 +0.711382,799,486.526034 +0.212714,1166,490.162396 +0.2813,1660,496.028137 +0.326661,1619,500.114207 +0.159701,1386,505.355108 +-0.071708,875,506.670864 +0.090959,1632,512.384051 +0.390612,2145,519.090096 +0.306258,1475,524.835735 +0.637869,1551,529.168142 +0.148495,1816,535.591778 +0.158515,1398,539.720874 +-0.348,734,542.705121 +-0.317,1255,546.595661 +-0.349,1100,549.926533 +0.119519,1733,555.513649 +0.000139,1527,559.493445 +-0.348,1476,564.903809 +0.246965,2563,571.899967 +0.006402,1444,577.138756 +0.183515,1366,580.770298 +0.093538,1366,584.71654 +-0.006942,1820,591.182403 +-0.345,1551,596.844327 +-0.167979,1610,601.199943 +-0.344,1215,606.500184 +-0.127189,1283,610.206978 +0.17023,1879,616.21545 +-0.08845,1104,619.447153 +-0.115312,1691,623.682969 +-0.351,1703,629.398786 +0.046971,1393,633.026876 +0.654291,1020,636.088574 +0.296042,1474,641.88836 +0.207589,1393,645.986554 +0.171515,859,649.180724 +0.161692,1004,652.591506 +-0.047345,1118,656.059297 +0.439674,1816,662.452899 +-0.119719,695,665.224286 +0.215357,1689,669.57638 +0.133807,1206,672.997355 +0.554569,1141,676.219561 +0.143711,1061,681.136019 +0.612138,1053,684.297927 +0.142124,1360,687.912632 +0.35594,1303,691.454805 +0.61183,1122,694.705117 +0.348773,1638,700.282369 +0.260138,1118,703.69381 +0.25022,1459,707.834751 +-0.136189,697,710.506485 +-0.092708,656,713.368481 +0.512038,1350,717.456492 +0.215936,1185,721.207569 +0.790766,763,724.188263 +-0.10633,949,727.460279 +0.500264,1197,730.916387 +0.154322,1561,736.378494 +0.084305,2713,743.596596 +0.288042,2210,749.929475 +0.271667,1939,755.80146 +-0.034282,1310,761.071845 +0.10565,2669,769.011494 +0.665054,1081,772.349292 +0.63155,967,775.723073 +0.268453,2127,861.25144 +-0.055274,1297,864.678903 +0.353421,3485,875.956086 +0.193866,2294,882.891786 +0.05049,1756,889.056097 +-0.170979,1113,892.521465 +0.392633,2891,902.21622 +0.201564,2048,909.130561 +-0.344,1215,912.500934 +0.346908,2427,919.333871 +0.124238,2442,927.647662 +0.430553,2160,934.037483 +0.568745,1695,938.161916 +-0.119719,1924,944.71899 +0.413274,2635,954.097798 +0.397897,2141,960.963854 +0.429475,2720,970.295653 +0.443202,1759,974.55729 +0.645459,1489,979.827667 +0.106703,2176,986.089545 +0.185866,2401,993.050671 +0.258843,1696,998.988194 +0.318404,2209,1006.034883 +0.41305,3116,1016.445231 +0.339808,2304,1023.746192 +0.204883,1337,1029.279184 +-0.012942,949,1032.342279 +0.47771,1950,1038.691317 +0.337773,1660,1042.72301 +0.354908,2174,1049.116896 +0.034518,1050,1052.316773 +0.212564,1974,1058.462567 +0.393509,1766,1064.790257 +-0.35,805,1067.635236 +0.291153,1923,1073.914248 +-0.300704,888,1075.434971 +0.344843,1183,1080.467153 +0.185059,1053,1083.905945 +0.039737,1199,1087.433287 +0.32958,1561,1091.687403 +-0.088675,799,1094.410797 +0.083305,1172,1097.669537 +0.195297,1281,1102.680112 +-0.11433,1210,1106.012133 +0.092668,1121,1109.231067 +0.067305,1096,1112.45206 +0.005402,1543,1116.243362 +0.218167,1378,1121.801496 +0.13565,1146,1125.317314 +0.445466,1546,1129.722465 +0.136807,1713,1135.968982 +0.034887,1681,1142.138187 +-0.35,1345,1146.098068 +-0.167979,1559,1150.017407 +0.038887,2163,1157.891188 +-0.343,719,1158.930604 +-0.003357,1401,1164.087668 +-0.135189,1181,1167.377803 +0.230488,1911,1173.445371 +0.04749,1768,1177.539036 +-0.076591,1939,1183.735026 +-0.119189,1125,1187.273913 +-0.063307,1855,1193.385057 +0.01055,1850,1199.738269 +-0.092118,1783,1205.542241 +-0.02251,1551,1209.611933 +0.611484,1124,1212.831633 +-0.053913,1365,1257.246516 +0.092668,1575,1261.672053 +-0.06606,1667,1267.971838 +0.113322,1713,1274.018948 +0.267378,1772,1278.298563 +-0.34,860,1281.19556 +0.35397,1934,1287.302738 +-0.125489,1623,1293.081073 +-0.103312,760,1294.226408 +0.07287,1550,1299.920285 +0.133059,1015,1303.459678 +-0.229111,807,1306.304986 +0.06687,1324,1310.216691 +-0.06685,1158,1313.787076 +0.014225,1988,1320.335264 +-0.237557,1135,1323.923253 +0.177277,920,1327.124238 +0.125926,1330,1330.766256 +0.279863,1644,1336.298713 +0.06187,1378,1339.90693 +-0.292704,1093,1343.077613 +-0.006942,889,1345.916135 +-0.344,776,1348.635816 +0.120238,1094,1351.759519 +0.033136,1439,1355.401 +0.475082,1500,1360.853333 +-0.05906,1527,1365.095499 +0.024305,1436,1370.650706 +0.227488,2269,1377.845103 +0.2593,1690,1384.237552 +-0.153979,802,1385.820739 +-0.155979,1520,1391.439341 +0.018518,2532,1398.59189 +0.412744,2411,1407.03341 +-0.050675,815,1409.792042 +0.151124,2447,1416.783543 +0.100028,2969,1453.420833 +-0.191557,749,1455.987653 +0.169124,2371,1462.494703 +0.136314,2130,1468.679571 +-0.209034,1199,1472.010032 +0.209297,2017,1478.133822 +0.284863,2671,1487.710338 +0.060421,1620,1492.245922 +0.197714,2107,1499.292209 +0.225589,2161,1506.251105 +-0.229301,918,1509.325238 +0.140519,1811,1515.233434 +-0.339,1453,1520.525162 +0.196557,1642,1524.52582 +-0.097133,1540,1529.974743 +-0.351,1278,1533.503356 +-0.208126,1153,1536.863022 +0.051971,1182,1540.376669 +-0.06406,1599,1546.170679 +0.116238,1560,1550.229015 +-0.003598,1490,1555.756008 +-0.111312,855,1558.675586 +0.126314,1832,1563.295238 +-0.35,1120,1566.929199 +0.055994,1710,1572.540122 +0.383589,2607,1581.04937 +-0.154159,1645,1585.038229 +-0.113719,819,1587.84268 +0.130807,1366,1591.416821 +0.137314,1313,1596.737995 +-0.297,698,1597.786206 +-0.194892,1224,1603.085769 +0.156322,7895,1627.861944 +0.163883,7513,1651.858899 +0.30845,3148,1661.703385 +-0.015771,6835,1683.042569 +0.347876,2796,1692.217295 +0.102703,1947,1698.087837 +0.302308,2414,1704.675275 +0.195297,7623,1729.387191 +0.655668,1418,1733.751585 +0.089139,1707,1740.039815 +-0.078603,1359,1744.177454 +0.487497,1597,1750.177523 +0.325098,1528,1753.965558 +-0.170979,1154,1757.263989 +0.096668,1793,1763.023611 +0.4838,1753,1768.853634 +0.314404,2207,1775.428243 +-0.040422,1819,1781.620307 +0.18123,1396,1785.539181 +0.029862,917,1788.71071 +0.463826,1825,1794.955051 +0.538331,1247,1798.604486 +0.257453,2063,1805.454289 +0.145495,1867,1811.581209 +0.484778,1621,1815.630368 +0.139314,1024,1818.82353 +-0.087861,934,1821.742221 +-0.148677,667,1824.269265 +0.014881,1068,1827.488811 +0.051484,1290,1830.936901 +0.605624,1089,1834.120461 +0.509868,1424,1839.431168 +0.111368,771,1840.782043 +-0.001598,976,1844.13509 +0.077538,1599,1850.156449 +0.513584,1488,1854.556118 +0.007139,971,1857.910368 +0.033881,857,1861.096752 +0.469259,1789,1867.344983 +0.592102,1198,1870.875237 +0.194297,1165,1874.116925 +0.311308,1661,1879.699814 +0.35197,1415,1883.317267 +0.03255,953,1886.222523 +-0.11133,834,1889.035276 +0.203816,1370,1892.819859 +0.011225,850,1895.886622 +0.200432,1353,1901.392485 +0.276799,1341,1905.13913 +0.012225,1168,1908.593209 +0.151272,1301,1912.287305 +0.188157,1447,1917.79455 +0.105023,1204,1921.299136 +0.083139,1292,1924.903875 +0.37697,2042,1931.232042 +-0.089708,882,1934.122911 +0.244965,1544,1937.943445 +0.414081,2320,1946.069438 +-0.208034,679,1947.042175 +0.447108,1636,1952.50968 +-0.256389,772,1955.200019 +-0.211029,857,1958.030758 +-0.044298,1058,1961.324808 +0.319538,1501,1965.475011 +-0.118489,939,1968.789795 +-0.141111,807,1971.751039 +0.338538,1243,1975.536555 +0.120792,1359,1979.473296 +0.200863,1974,1985.968439 +-0.173791,619,1988.612457 +0.427553,2118,1995.086241 +0.117028,1333,1998.80877 +-0.346,1115,2002.186974 +0.531946,1542,2007.851241 +-0.014295,1005,2011.074259 +0.097495,2041,2017.472274 +0.687268,1091,2020.768461 +0.498429,1884,2027.204569 +0.25022,3266,2037.528166 +0.520224,1909,2043.977005 +0.329737,1994,2050.595116 +0.501063,2084,2056.844635 +0.225384,1834,2062.8128 +0.052971,2058,2069.282586 +0.435135,2240,2076.121193 +0.457179,2495,2083.235508 +0.635026,1162,2086.889832 +0.717344,963,2090.244895 +0.602459,1489,2096.052326 +0.6952,1155,2099.615784 +0.428253,2402,2107.121103 +0.272,2086,2113.674139 +0.351876,2131,2120.222419 +0.289984,1965,2126.823019 +-0.069307,793,2129.66934 +-0.244557,802,2132.53505 +-0.006138,828,2135.333235 +-0.140121,967,2138.506217 +0.137792,1251,2142.089 +0.034737,1114,2145.5387 +0.326404,1204,2149.103445 +0.094368,1307,2152.843631 +0.528632,1258,2158.06034 +0.205432,1399,2161.957318 +0.315308,1354,2165.642179 +-0.029295,993,2168.819197 +0.159322,904,2172.02226 +-0.351,643,2174.630611 +-0.04785,775,2175.833546 +0.122344,1205,2180.995522 +0.122238,1090,2184.240072 +-0.161791,734,2185.333263 +-0.056307,681,2188.065522 +-0.082274,967,2191.122065 +-0.220288,690,2193.761276 +-0.286704,627,2194.751048 +0.03932,737,2197.503807 +0.085538,860,2200.60533 +0.11065,933,2203.703801 +0.279863,1149,2207.215951 +0.687173,886,2210.298792 +0.126314,1243,2214.004173 +0.036887,748,2216.829189 +-0.11133,1037,2220.229309 +-0.002972,1033,2223.378472 +0.35597,1768,2228.107189 +-0.059674,1375,2233.456574 +0.293206,1206,2236.896558 +-0.337,1252,2240.377609 +0.025518,1555,2245.776022 +0.037136,1341,2249.390244 +-0.352,1311,2252.85374 +0.575432,1279,2257.928503 +0.420331,1498,2261.944631 +0.504647,1587,2267.729225 +-0.348,1403,2271.508476 +-0.349,1353,2275.237856 +0.115238,1267,2278.851226 +-0.274416,1111,2283.686707 +-0.120719,1511,2287.573651 +0.547995,1302,2291.117469 +-0.295452,1227,2296.017473 +-0.288704,1080,2299.180702 +-0.294704,827,2300.431997 +0.265598,1621,2306.026564 +0.168515,1691,2311.735467 +-0.149677,934,2314.787672 +-0.352,1120,2318.100249 +0.067421,1736,2322.57612 +0.064421,1447,2328.246228 +0.617912,1394,2332.105186 +0.06287,2284,2339.001438 +0.378632,2311,2347.540457 +0.456259,1789,2351.846385 +0.123057,2070,2358.245325 +0.454913,2086,2364.53026 +0.228277,1957,2370.620599 +0.437388,2699,2379.421379 +0.423075,2827,2389.261088 +0.540823,1402,2393.273911 +0.360116,1645,2399.271965 +0.515053,1982,2405.830998 +0.652625,1360,2409.624835 +0.451607,1882,2415.672627 +0.194866,2414,2422.739834 +-0.144121,2417,2429.647982 +0.434744,2888,2438.692997 +0.305308,2275,2447.180055 +0.451535,1697,2451.691131 +0.36597,2685,2460.792165 +0.565904,1762,2466.928705 +0.401682,2430,2474.079058 +0.693398,1142,2477.451829 +0.296153,2286,2483.914983 +0.471276,1897,2489.859246 +-0.102133,2544,2498.480558 +0.281924,3162,2508.298685 +0.126314,3250,2518.200438 +0.590095,1400,2522.064434 +0.40156,2296,2528.932363 +0.063484,2784,2537.964735 +-0.082913,1084,2541.065702 +0.237965,2263,2547.63395 +0.51688,1526,2553.192864 +0.256378,2145,2559.771576 +-0.339,1403,2563.756909 +-0.162979,1475,2569.340636 +0.291734,1906,2575.597026 +0.242965,1899,2581.990884 +0.194432,2069,2588.556953 +0.121519,1817,2592.86339 +0.141138,2368,2601.154252 +0.192014,1750,2605.458728 +-0.055531,1888,2611.732713 +0.260453,1991,2618.190382 +-0.014771,1681,2624.12154 +0.176396,1903,2630.400622 +-0.246389,1147,2633.858814 +0.151792,1794,2639.934074 +0.324661,1867,2644.572418 +0.115238,2231,2652.856388 +-0.139121,1226,2656.260673 +-0.059674,1529,2660.022514 +0.039971,1059,2663.096704 +-0.000357,1188,2666.386111 +-0.147159,1067,2669.574053 +0.192297,943,2672.458737 +0.222488,1917,2678.399758 +0.428981,1935,2685.00839 +0.31445,1804,2691.304879 +0.241965,1656,2697.560014 +0.031737,1523,2701.772791 +0.44284,2833,2711.319422 +0.206564,1650,2715.386133 +0.127564,2087,2721.601892 +0.17323,1564,2727.023172 +0.161314,1232,2730.340679 +0.497543,1434,2735.662864 +0.455735,1724,2739.962049 +0.156124,1668,2746.015464 +-0.028422,932,2749.184219 +-0.001876,1194,2752.851671 +0.253688,1442,2757.124867 +0.263527,1175,2760.693759 +0.190866,1458,2766.377079 +0.304357,1314,2770.119318 +0.422967,2258,2776.608667 +0.504753,1440,2782.113143 +-0.073591,805,2784.937071 +-0.238288,773,2786.063277 +-0.012406,1245,2791.055039 +-0.128489,844,2792.326206 +0.513919,1559,2797.644229 +0.027518,1016,2800.987561 +0.268783,1689,2807.365137 +0.35294,2058,2814.253692 +-0.011502,1387,2818.460335 +0.226488,1246,2822.291502 +0.11265,1239,2826.085949 +-0.193557,693,2828.781463 +-0.271113,632,2831.303579 +-0.023282,832,2832.554826 +0.042437,1301,2837.682731 +0.305404,1488,2841.604492 +0.500698,1705,2847.330455 +-0.011345,851,2850.180883 +0.236053,2036,2856.466957 +-0.139268,864,2859.30354 +0.322153,1743,2863.914776 +0.510107,1740,2870.072052 +-0.048531,970,2873.28235 +-0.177791,1207,2877.031291 +0.312495,2397,2884.425658 +0.350843,2364,2892.88654 +-0.184304,1328,2896.377923 +0.452793,1973,2902.579938 +0.293984,1348,2906.090774 +-0.013623,751,2908.778114 +0.4758,1753,2914.48339 +0.31545,1682,2918.672555 +0.675359,1166,2922.307965 +0.329538,1887,2928.983038 +0.35094,1505,2934.774523 +0.57838,1148,2938.479162 +-0.028531,985,2941.782721 +0.627229,1287,2945.639266 +0.045971,1123,2949.057375 +0.203816,1847,2954.91042 +0.466914,1656,2960.479946 +0.393068,2196,2966.781787 +0.35597,2504,2973.60168 +0.067305,2019,2979.933225 +0.241816,2031,2986.60401 +0.33758,1633,2992.632631 +0.369808,2546,3000.190516 +0.2613,2593,3009.412855 +0.069728,1418,3013.025114 +0.063484,1443,3018.213574 +0.583209,1216,3021.53969 +0.194277,1326,3025.111205 +0.110936,1430,3028.796881 +0.520925,2169,3036.724009 +0.313098,2427,3044.000988 +0.125948,1360,3048.09879 +0.322621,1831,3054.561651 +0.462529,1807,3061.111567 +-0.344,1495,3065.496917 +0.206816,1705,3071.320697 +0.364725,2102,3077.597301 +-0.046531,1364,3081.150166 +0.398621,2740,3089.972838 +-0.333,987,3092.994737 +-0.17435,1615,3098.481955 +0.35594,3177,3108.769684 +-0.17835,1285,3112.589507 +-0.352,2091,3119.323831 +0.201297,1513,3123.609362 +-0.144111,1670,3129.480162 +-0.098603,1338,3133.017705 +-0.190113,1023,3136.056146 +-0.208288,1689,3141.585299 +-0.080913,1277,3144.967193 +-0.154979,2097,3151.174063 +0.439069,2241,3157.54627 +0.122792,1294,3163.002219 +-0.337,1547,3167.45009 +0.231688,1561,3173.375393 +0.017881,1528,3177.827719 +0.262688,1156,3181.549437 +0.281984,2516,3190.784662 +0.564675,1433,3194.42578 +-0.229113,1417,3199.68172 +-0.083913,1341,3203.253879 +0.125314,1345,3206.806545 +0.223488,2277,3213.423938 +-0.120489,1348,3218.658591 +-0.082913,1365,3222.61344 +-0.34,597,3225.13959 +-0.092118,972,3228.477257 +-0.349,975,3231.55606 +-0.024295,1952,3238.119529 +-0.15044,883,3239.546431 +0.359773,2149,3246.355235 +0.034737,1512,3251.800153 +0.446564,2280,3258.244445 +-0.35,863,3261.114645 +0.234688,2823,3269.886382 +0.114948,1969,3276.003557 +0.315495,2570,3283.490263 +0.210816,1853,3289.932563 +0.098368,1715,3296.336874 +0.278924,2516,3304.138137 +-0.308,317,3304.769348 +0.328308,2001,3311.275757 +0.287734,2643,3320.057507 +0.177557,1682,3325.831429 +0.037855,1500,3329.56628 +0.150921,1793,3335.320864 +0.225297,1551,3339.28311 +0.009855,1872,3345.590387 +0.633049,1217,3349.292682 +0.099421,1076,3352.617174 +0.118238,2373,3361.131613 +0.50238,1466,3365.125313 +-0.140111,1581,3370.752714 +-0.316,545,3371.558929 +-0.171979,1207,3374.865713 +-0.013771,1511,3380.299099 +0.708798,1044,3383.453904 +0.183866,1737,3389.120378 +-0.300704,807,3390.265381 +-0.016771,1557,3395.646373 +-0.230113,1728,3401.786225 +-0.352,926,3404.976938 +0.105538,1664,3409.589694 +0.038518,1626,3415.68873 +-0.15335,1208,3419.251427 +-0.171979,1461,3423.486032 +0.305308,2071,3429.916502 +0.333357,1735,3435.67414 +0.232783,1747,3441.44432 +-0.080274,1353,3445.062459 +-0.081274,1108,3448.199685 +0.173396,2286,3454.768259 +-0.038345,1321,3459.907258 +0.143792,1439,3464.064329 +0.318153,2117,3471.082403 +-0.229113,784,3474.053053 +0.014862,964,3477.279507 +0.266378,1872,3483.78151 +0.247404,1506,3488.044053 +-0.084274,1378,3493.261812 +0.36694,1573,3497.150517 +0.155495,1302,3500.618355 +0.052994,1288,3505.596302 +0.599909,1139,3508.824994 +-0.000598,980,3511.804973 +0.006862,1077,3514.949461 +-0.339,646,3515.886644 +0.119792,1256,3521.020645 +-0.224288,571,3522.069886 +0.305206,1568,3527.818827 +-0.218034,934,3531.152563 +0.432715,1734,3535.807103 +0.107926,577,3538.442038 +-0.242923,852,3541.440731 +0.307153,1484,3545.700565 +0.177557,1879,3551.837416 +-0.343,1472,3555.720024 +0.334737,2089,3562.051807 +-0.15135,789,3564.884984 +0.337808,2260,3571.442088 +0.347876,1378,3576.664911 +0.353908,1637,3580.931627 +-0.148268,1047,3584.207389 +0.44747,1941,3590.635359 +0.39994,2258,3597.524659 +0.215863,1750,3603.597916 +-0.343,809,3606.507513 +0.583607,1411,3610.35376 +0.537735,1519,3615.79419 +0.243816,1386,3619.366106 +0.341876,2115,3625.580215 +0.175396,2014,3631.675213 +0.503246,1523,3637.081958 +-0.010138,884,3639.977059 +0.512399,2382,3647.285137 +0.089926,1198,3650.987665 +0.000887,829,3654.052321 +0.023136,1352,3658.099741 +0.059421,1121,3661.608147 +0.471299,1979,3668.386918 +-0.220923,738,3671.042192 +-0.277416,1457,3674.802297 +-0.345,1397,3679.963904 +-0.343,986,3683.039049 +-0.112719,2047,3689.220487 +0.172396,2039,3695.388716 +-0.115,1049,3698.542931 +-0.140111,1991,3704.842278 +-0.082913,1470,3709.022272 +0.499119,1682,3714.848963 +-0.069591,1705,3719.415927 +-0.17335,1217,3724.658821 +-0.107489,825,3726.189549 +-0.037295,1519,3731.586523 +-0.051674,1490,3735.39761 +-0.268632,1636,3740.910265 +0.088023,1425,3744.496333 +0.338808,2793,3753.175724 +-0.17435,1284,3756.63737 +0.492551,1632,3762.657962 +-0.001138,1108,3766.223657 +-0.034295,1765,3772.504371 +0.718076,783,3773.979744 +0.125314,1295,3779.611264 +-0.155677,991,3783.012228 +0.580041,1180,3786.850374 +0.194515,1176,3790.523458 +0.06649,954,3793.553183 +0.060971,1146,3796.84827 +0.046887,1233,3800.295167 +-0.341,1393,3805.447176 +0.218357,1266,3808.851801 +-0.004357,1019,3811.894235 +0.127564,2373,3818.640186 +0.533984,1889,3825.274294 +-0.35,1189,3828.830425 +0.06287,1964,3835.4508 +0.192866,1896,3841.940026 +-0.15544,2002,3848.532753 +-0.084274,1082,3851.686799 +-0.088274,1155,3854.994491 +0.109948,2601,3863.464798 +-0.129489,1210,3866.816376 +-0.341,1011,3869.86583 +-0.085274,1079,3872.982919 +0.45966,2232,3879.504321 +0.158921,1463,3883.872256 +0.358773,1925,3890.510958 +-0.291704,656,3893.280232 +0.243965,2445,3901.238082 +0.200297,2432,3910.450869 +0.36194,2939,3919.460565 +0.075139,2916,3927.094302 +0.128564,2117,3935.037429 +-0.11533,1606,3939.164108 +0.241965,2715,3948.560927 +0.11665,2095,3955.413003 +0.206692,1615,3960.066627 +0.37897,1749,3966.30495 +0.285799,2546,3974.766552 +0.173396,1683,3978.838822 +0.642726,1039,3981.896389 +0.584229,1287,3985.363463 +0.382624,2159,3993.313566 +0.256453,2423,4000.115316 +0.053437,1543,4004.543534 +0.467788,1914,4011.188746 +0.276378,1445,4016.994166 +0.310308,1895,4023.664815 +0.331737,1781,4028.496334 +0.528688,1403,4033.597858 +0.267527,2141,4039.780754 +0.216167,2034,4045.811713 +0.586264,1662,4049.781416 +0.38494,2743,4058.601798 +0.540469,1427,4062.616107 +0.415879,2060,4069.292865 +0.078728,2315,4077.892129 +0.121792,1758,4082.579987 +0.348808,1962,4088.91525 +0.077538,3420,4100.187304 +0.107124,2944,4109.122356 +0.627523,1118,4112.372732 +-0.058307,3121,4122.038049 +0.263453,3215,4132.637727 +0.091368,2126,4139.800949 +0.092023,2628,4149.231755 +0.135124,1977,4153.945537 +-0.326,2254,4162.173186 +0.272378,3024,4171.674757 +0.360661,2468,4178.411371 +0.250138,2734,4187.781152 +-0.168677,2953,4197.581732 +0.122519,2988,4205.939325 +-0.019623,2874,4214.808701 +0.028136,2100,4221.03255 +0.243053,2926,4230.009484 +0.278799,2021,4236.103949 +0.022136,2324,4243.396444 +0.070305,2280,4252.153447 +0.138272,1848,4256.855669 +0.01655,1946,4263.384491 +0.302153,2972,4272.531811 +0.275863,2486,4281.078846 +0.582089,1484,4284.818163 +0.303042,3449,4296.260442 +-0.09733,1037,4299.400427 +0.293098,1531,4303.582474 +0.289924,1155,4307.088681 +0.011855,719,4309.788829 +-0.030406,1113,4313.327255 +-0.033531,840,4316.193095 +0.515871,1387,4320.181714 +0.237688,1224,4324.063946 +0.041971,1073,4327.614148 +0.024855,853,4330.48459 +-0.343,602,4332.915373 +-0.136189,960,4335.865295 +0.472231,1779,4340.069151 +0.582238,1015,4343.209347 +0.316357,793,4345.914411 +0.35,2062,4352.030387 +0.107948,1450,4357.337312 +0.056959,1026,4360.542578 +0.102926,775,4361.881787 +0.570102,1198,4367.306204 +0.26022,1311,4371.251237 +0.080926,1113,4374.803058 +-0.057674,976,4378.258809 +0.024225,993,4381.609133 +-0.009406,1086,4385.15683 +0.091368,927,4388.395686 +0.098028,1555,4392.267024 +-0.062675,699,4394.850562 +0.222488,3345,4404.539643 +-0.349,1355,4409.71288 +-0.339,782,4410.90106 +-0.255958,1157,4415.738795 +0.51739,1798,4420.049739 +0.029518,1110,4423.513936 +0.229384,3315,4435.338314 +0.602729,1188,4438.806216 +0.01255,1300,4442.455612 +-0.061674,1969,4448.963027 +-0.009138,2008,4455.003041 +-0.13944,935,4457.869851 +0.143711,1370,4461.445308 +0.547135,3644,4473.043081 +0.043971,1217,4476.369048 +-0.322,838,4479.229374 +0.601368,1215,4482.945704 +-0.05885,1226,4486.804984 +-0.068307,1335,4490.725798 +0.03432,1499,4496.622129 +0.45594,1722,4503.001523 +-0.345,464,4503.876885 +-0.084675,1210,4507.723686 +-0.169677,988,4510.870716 +-0.137121,629,4513.433221 +0.06687,1365,4516.965324 +0.172314,1654,4522.595638 +0.065728,992,4525.609026 +0.166883,1816,4529.901157 +0.307357,1786,4535.635861 +-0.147489,533,4538.05758 +0.260053,1234,4541.791729 +0.48753,1726,4547.805227 +0.07187,1160,4551.417017 +0.155124,1151,4555.163406 +0.182701,1157,4558.76853 +0.442519,2026,4565.444977 +0.335661,1831,4571.528058 +-0.102708,602,4572.412058 +0.134564,1605,4577.990885 +0.186396,1227,4581.417635 +0.127238,1146,4584.699157 +0.152432,2323,4592.871074 +0.442751,1194,4596.214182 +-0.349,505,4596.968276 +0.052994,1077,4600.354687 +0.075421,1487,4606.417712 +-0.080913,1568,4610.765579 +0.217277,2081,4617.724814 +0.36,3390,4630.20541 +0.229384,3320,4639.850678 +0.63153,1003,4643.032104 +0.134057,1269,4646.524737 +0.033737,1433,4650.19557 +-0.001357,1118,4654.93996 +0.023881,1039,4658.119172 +0.242153,1625,4662.429227 +0.314357,1499,4668.294024 +-0.162677,1630,4672.560985 +0.158124,2192,4679.436545 +0.232589,1705,4685.43347 +0.505538,2397,4692.441133 +-0.088274,1050,4695.57734 +-0.294452,1524,4700.914611 +0.302357,2650,4707.987097 +0.146924,1612,4713.487637 +0.538094,1566,4718.920379 +0.232564,1895,4723.806137 +0.474347,1547,4729.751319 +0.322357,1338,4733.839597 +0.556109,1583,4739.783026 +0.161124,1502,4744.106295 +-0.049345,1109,4747.727468 +0.198432,1450,4753.107336 +0.157807,1779,4757.26502 +0.178783,1554,4762.725338 +0.427677,2024,4768.967131 +0.301042,3067,4778.25025 +0.343876,2706,4787.59367 +0.508911,1803,4792.24572 +0.309258,1854,4798.4222 +0.382401,1757,4804.503742 +0.213167,2570,4811.662571 +-0.081913,1022,4814.718168 +-0.183791,1181,4819.629143 +0.033136,1743,4823.761681 +-0.148111,1382,4828.93657 +-0.342,936,4830.329538 +-0.35,811,4833.109017 +-0.225288,1028,4836.310459 +-0.190557,942,4839.450267 +-0.14735,990,4842.822691 +-0.122719,1050,4846.313598 +-0.348,1008,4849.708977 +-0.126312,770,4852.710364 +0.089057,1150,4856.474129 +-0.255557,914,4859.662224 +0.073728,1194,4863.471691 +-0.336,789,4866.549375 +-0.018771,1456,4870.605457 +-0.345,1060,4873.815956 +-0.344,576,4876.247538 +0.200432,1698,4881.887199 +-0.35,1197,4885.314188 +-0.001598,2016,4891.558918 +0.161272,1298,4895.047681 +-0.143979,886,4897.995451 +0.161515,1337,4901.939455 +-0.041675,755,4904.764251 +-0.006771,1373,4908.78973 +0.03332,1014,4912.075023 +0.329928,3120,4922.47082 +0.451788,2215,4929.251211 +0.246138,2117,4935.597492 +0.556559,1265,4940.617594 +0.015229,1317,4944.124383 +0.33197,2480,4950.951009 +0.446108,1627,4956.539521 +0.113344,2560,4965.952591 +0.543089,1508,4970.130159 +0.443137,2110,4977.393052 +0.144043,1976,4984.05484 +0.208936,1746,4990.474055 +0.479386,2315,4997.185249 +0.651049,1217,5000.619446 +0.46602,1731,5006.397124 +-0.022623,1482,5010.276958 +0.348484,2250,5016.935491 +0.150124,1914,5023.409482 +-0.060674,2172,5030.437284 +0.103703,2142,5038.790777 +0.05449,1952,5043.603817 +0.035887,1662,5049.392589 +0.119519,2212,5055.852248 +0.487878,1585,5061.384028 +0.565473,1412,5065.045184 +0.433203,2318,5073.577682 +-0.028337,1420,5078.811265 +-0.344,1513,5082.68572 +0.46533,1355,5088.259898 +0.649847,1303,5092.252929 +0.337773,1496,5096.432387 +0.454186,2056,5103.344823 +0.04332,2148,5110.317121 +0.235688,2003,5116.542571 +0.197866,2427,5124.808918 +0.456213,1769,5130.577841 +0.633023,1461,5134.270043 +0.251138,3800,5146.810488 +0.003855,3173,5156.515974 +0.066043,2000,5163.071823 +0.177297,1850,5167.678393 +-0.081708,2320,5175.919214 +0.172396,2657,5183.013928 +0.436837,2324,5191.158984 +0.246053,2385,5197.767512 +0.134028,2452,5205.347904 +0.114238,1706,5211.622996 +0.633832,1231,5215.307814 +0.35097,8183,5241.836862 +0.052484,1563,5247.295573 +0.277863,1678,5251.421053 +-0.347,868,5254.312518 +0.095703,1388,5259.4826 +-0.235288,961,5262.566366 +0.04949,2371,5269.91487 +0.284799,1671,5274.450185 +0.190557,1562,5280.412756 +-0.231389,1509,5284.568798 +0.624338,1004,5287.815119 +-0.101708,1507,5293.229229 +0.445113,1378,5296.868172 +0.141792,1340,5301.986547 +0.365808,1978,5307.96617 +0.311495,1932,5312.380959 +-0.34,990,5315.416383 +0.462549,2013,5321.715591 +0.470668,1456,5327.471761 +-0.076133,1828,5334.058954 +0.234783,1516,5338.380817 +-0.16344,1211,5342.226767 +0.00855,2547,5351.701293 +-0.097603,974,5354.72539 +0.468324,2207,5361.158746 +-0.15644,1302,5364.720943 +-0.289704,1129,5368.058847 +0.345876,1982,5374.211908 +0.333773,2062,5380.403269 +-0.165677,1201,5383.887744 +-0.090603,1539,5389.677858 +-0.115312,1452,5393.813883 +0.072305,1336,5399.16584 +0.223488,1957,5404.410425 +0.28522,1319,5409.64714 +0.053959,1454,5413.574143 +0.313357,2331,5420.10354 +0.254688,1774,5425.868834 +0.287042,1899,5431.8196 +-0.305704,833,5434.562989 +0.007643,1257,5437.963905 +-0.236557,1738,5444.041103 +-0.342,846,5447.189602 +-0.089118,1629,5451.649841 +0.273734,1427,5457.502013 +0.001377,1446,5461.660393 +0.183714,1718,5467.846429 +0.029881,1520,5471.943972 +-0.348,883,5474.79476 +-0.344,1405,5478.446155 +0.163883,2169,5484.679308 +-0.155979,1071,5489.325063 +-0.344,1315,5492.84808 +-0.180892,1114,5496.049621 +-0.168979,1149,5499.358802 +0.329495,1622,5504.890898 +0.209816,1447,5508.935915 +0.124314,2245,5515.931125 +0.25122,1569,5521.639646 +-0.352,1326,5525.41189 +-0.247923,1546,5531.185856 +0.021881,1561,5535.062219 +0.048437,1627,5540.574497 +0.268527,2047,5546.653633 +0.62441,1458,5550.391278 +0.175396,1441,5554.089336 +0.181557,1824,5559.906384 +0.100368,1608,5565.732669 +0.152272,1288,5569.830205 +0.244053,2201,5577.081988 +0.426455,1965,5583.899012 +0.559722,1479,5589.855785 +0.273734,1854,5594.169427 +0.728385,933,5597.016069 +0.300206,1905,5602.922627 +0.269527,1706,5608.621586 +-0.021502,1775,5614.383343 +0.284042,2316,5621.013254 +0.273667,2118,5627.808437 +0.108948,1377,5631.948876 +0.189157,1777,5638.117679 +0.226167,2277,5645.33524 +0.111238,1827,5651.563409 +0.298984,1735,5657.263985 +0.295206,1876,5663.18263 +-0.237288,1518,5666.924691 +0.212692,1840,5672.686523 +0.351737,3236,5682.351998 +0.25422,3225,5692.922391 +0.2563,2129,5699.847957 +0.064994,1857,5706.378307 +0.115948,1695,5712.485779 +-0.225029,995,5715.507817 +0.608865,1175,5718.824531 +0.201564,1510,5722.678394 +0.324153,2170,5729.11362 +-0.119719,975,5732.188314 +0.113948,2162,5738.653415 +0.252138,2095,5745.403173 +0.388498,2297,5754.166664 +0.236876,2534,5761.839545 +0.541865,1675,5767.990386 +0.354908,1952,5774.219398 +-0.100298,965,5777.181948 +-0.119,1107,5780.424902 +-0.238288,1557,5784.401842 +-0.341,951,5787.374677 +0.223277,1256,5790.893337 +0.167059,1658,5796.536301 +0.469868,1470,5800.250466 +0.208816,2130,5808.846785 +0.058959,1471,5813.043015 +-0.116312,1203,5816.751998 +0.072421,2155,5823.944074 +-0.097133,1308,5829.629675 +-0.028337,1148,5833.070222 +0.596338,1004,5836.111869 +0.153124,1887,5842.126812 +-0.15644,836,5843.376813 +0.027737,2205,5849.906301 +0.292863,2113,5857.717624 +-0.343,555,5858.537487 +0.193157,1485,5864.28233 +0.290098,1976,5869.197867 +0.151138,1511,5874.913371 +-0.050531,776,5877.794555 +-0.104312,514,5878.666167 +-0.016771,1076,5882.286914 +0.214936,1932,5888.820822 +0.12065,986,5892.042545 +0.245384,1394,5895.700425 +0.309,1638,5901.286556 +-0.006771,1279,5904.78494 +-0.015422,1042,5907.948572 +0.234384,1570,5913.406626 +-0.033295,921,5916.339752 +0.516463,1227,5919.733629 +0.055959,1561,5924.19836 +-0.35,557,5926.894445 +-0.138189,1063,5930.482134 +0.247053,2249,5937.994894 +-0.196892,1324,5943.553428 +0.074305,935,5945.37597 +-0.086274,1245,5950.9072 +0.070421,7492,5973.094405 +0.195157,1480,5976.891176 +0.146495,1181,5981.796896 +0.451652,2395,5989.174185 +0.198014,2390,5996.509364 +-0.13044,1540,6002.370263 +0.16923,2656,6010.234923 +0.007855,3165,6019.7185 +0.286734,3149,6030.614329 +0.377325,3140,6039.839199 +0.429135,2638,6047.883652 +0.351564,2280,6055.188777 +0.445829,2128,6062.36235 +-0.025298,1636,6068.537771 +0.141238,1577,6074.141933 +-0.080913,2043,6080.372906 +0.31958,7121,6102.188617 +0.529012,1408,6106.241162 +0.336808,2866,6115.522102 +0.029518,1970,6122.184685 +0.289042,2568,6129.684911 +0.310357,3490,6141.224181 +0.343843,2604,6148.205766 +0.334917,2481,6156.544557 +0.277598,3575,6167.373222 +0.394474,2568,6176.643057 +0.465817,2276,6183.830182 +0.344166,2554,6191.403542 +0.397813,2502,6199.971796 +0.303258,3375,6209.791496 +0.096368,2534,6216.677556 +0.24922,2740,6225.640627 +0.374588,2814,6234.710389 +-0.032771,2233,6241.685884 +0.31858,2962,6251.55284 +0.472535,1697,6257.153012 +0.35097,2352,6263.787839 +0.373852,3511,6273.761007 +-0.352,829,6276.560491 +0.098926,1009,6279.650259 +0.486574,2064,6286.437614 +0.311357,1477,6292.225999 +-0.341,670,6293.46533 +-0.044531,895,6296.666902 +-0.336,892,6299.846632 +0.089305,938,6303.058818 +-0.073118,1492,6307.241227 +0.325538,1650,6313.058896 +0.318357,1429,6316.849548 +0.234688,2234,6324.885667 +0.431146,1720,6329.054803 +0.014023,1702,6334.842205 +-0.327,651,6337.634974 +0.458709,2011,6344.099353 +0.397297,1820,6348.988579 +0.489816,1733,6355.250047 +-0.147159,1064,6358.663886 +0.060437,1631,6364.557702 +0.5251,1750,6369.238692 +0.00955,1443,6374.598927 +0.092302,3259,6384.296524 +0.30945,1895,6390.245358 +0.338843,2274,6396.780075 +0.104344,2609,6405.906445 +0.189014,2605,6413.838614 +0.489705,1962,6420.686162 +0.151043,2051,6427.65946 +0.432204,2030,6434.263956 +0.026139,1256,6437.68719 +0.655058,1144,6440.956564 +0.541457,1580,6446.468524 +0.270453,2126,6452.855658 +0.503722,1479,6456.652216 +0.464781,2887,6466.161268 +0.377565,1562,6472.150282 +0.36197,1537,6476.384704 +-0.063274,937,6479.56561 +-0.002502,909,6482.523817 +-0.088603,1093,6485.898084 +0.021518,1530,6491.604765 +-0.038913,813,6492.783138 +-0.077913,1130,6496.073145 +0.463053,1982,6502.188851 +0.564461,1505,6507.592048 +0.360908,2358,6514.331911 +0.33645,2239,6520.904118 +-0.047422,677,6523.655755 +0.298153,2189,6530.829952 +-0.079591,1433,6536.497303 +0.148314,1272,6540.428329 +0.097368,1150,6543.945122 +0.547043,1721,6550.246143 +0.192014,1407,6553.891604 +0.49635,2003,6560.156629 +0.076421,1249,6563.576739 +0.592103,1760,6569.341666 +-0.017771,1326,6572.877005 +0.150272,3104,6582.269182 +0.329495,2445,6591.093794 +0.295892,3826,6601.948863 +-0.067482,1650,6607.867417 +0.565975,2154,6614.304911 +0.295206,3417,6624.173096 +0.598965,1845,6630.09135 +0.464646,2838,6638.889792 +0.324751,2915,6648.599843 +0.172515,2597,6656.481103 +0.359843,2665,6666.107239 +0.450413,2162,6672.819482 +-0.07106,2865,6681.971034 +0.603872,810,6684.721881 +0.636322,1052,6687.815156 +0.627133,1075,6690.944474 +0.087538,1612,6694.970953 +0.143711,1930,6701.164314 +0.221053,3053,6711.196565 +0.043887,1747,6717.278919 +0.291098,2581,6725.042861 +0.192866,2549,6733.723555 +-0.296704,1529,6737.594664 +0.071139,1494,6742.867958 +-0.352,1306,6746.272062 +0.193297,1574,6751.694504 +0.199432,2401,6758.373966 +0.008402,1408,6762.281101 +-0.334,1214,6767.613314 +-0.106719,1163,6771.35154 +0.594101,1268,6775.188399 +-0.343,1054,6778.793913 +0.479674,1816,6785.172569 +-0.05385,2434,6792.826478 +-0.34,862,6795.691527 +-0.077913,747,6798.600601 +-0.347,1097,6801.823327 +0.126238,2024,6808.056642 +-0.335,697,6809.125969 +0.01655,1412,6814.514986 +0.200692,1661,6818.732883 +0.434825,2142,6827.10932 +-0.129312,584,6828.132037 +0.190322,894,6831.254365 +0.331737,2224,6838.150245 +0.329621,2767,6847.371639 +0.122057,1456,6851.338668 +0.176059,1274,6856.438951 +-0.193159,847,6857.703573 +-0.141121,1531,6863.144211 +0.253488,1144,6866.426025 +-0.022406,849,6869.371567 +-0.230113,1195,6872.695554 +0.187014,1909,6878.739926 +0.095139,1856,6885.017663 +-0.117,788,6886.374966 +0.257453,1445,6892.043072 +0.149807,1545,6896.542332 +0.423924,1924,6903.069518 +0.305699,1175,6906.786506 +0.349908,2237,6915.146006 +0.753574,559,6915.985007 +0.506082,1500,6921.341305 +0.483918,1732,6925.645544 +0.150921,918,6928.635748 +0.382988,1550,6934.107512 +0.155322,1580,6938.047634 +-0.044345,1224,6941.680169 +0.154272,1209,6946.892132 +0.033225,1210,6950.502258 +0.243773,1886,6956.868342 +0.276667,1500,6961.018618 +0.250783,1491,6965.095321 +0.067484,1143,6970.147074 +0.191059,1323,6973.640304 +0.152404,1318,6977.159902 +0.232783,1968,6983.21078 +-0.110719,765,6985.961497 +0.06587,2748,6994.77563 +0.012225,2724,7002.175253 +0.45809,3051,7012.288314 +0.039887,2928,7022.021594 +0.390462,2923,7031.668286 +-0.041282,2718,7040.374459 +0.375572,3099,7049.693112 +-0.247923,2430,7056.456709 +-0.16044,2112,7063.000013 +0.261053,2563,7072.4312 +0.121057,2610,7080.457869 +0.415628,2525,7089.888759 +-0.012502,766,7091.021937 +-0.030337,1025,7094.078752 +0.086728,799,7096.835955 +0.813619,682,7099.422627 +0.166883,1165,7102.722572 +-0.005942,1093,7105.990694 +-0.337,763,7108.714064 +0.023881,1098,7111.959539 +0.336699,1179,7115.302588 +-0.091118,754,7118.009159 +0.302258,1265,7121.52588 +0.026881,851,7124.612368 +0.264153,1395,7128.825383 +-0.027282,1002,7132.283046 +-0.35,658,7135.001756 +0.097023,997,7138.365637 +0.731126,543,7139.373637 +0.253688,1083,7143.007174 +-0.261958,875,7146.018109 +-0.010138,722,7148.921973 +0.137043,1143,7152.590885 +0.205816,1285,7156.048967 +-0.260452,807,7158.821968 +0.253042,1405,7162.689427 +0.623587,1056,7165.834158 +0.222053,1113,7169.12726 +0.10565,1108,7174.105973 +0.312357,1375,7177.751518 +0.157564,783,7180.487637 +0.190059,892,7183.732581 +-0.139111,699,7184.957763 +0.129519,1231,7188.630695 +-0.004138,909,7191.749587 +-0.047422,800,7194.758445 +0.221167,2222,7201.689299 +0.35494,3531,7214.07029 +0.060959,2557,7220.994854 +0.293206,3772,7232.801884 +0.03055,2564,7239.710421 +-0.103,1760,7245.756227 +0.107344,2829,7255.530724 +0.119238,1996,7262.154307 +0.538182,2137,7269.298527 +0.338843,2836,7278.39509 +0.344737,3144,7287.794695 +0.126314,2827,7295.27331 +0.191322,1744,7301.041757 +0.144564,1472,7306.695998 +0.384373,3400,7317.233634 +0.023136,2158,7323.840893 +-0.001942,1586,7329.729916 +0.072305,3493,7339.554313 +0.303258,2527,7346.327104 +0.17023,2356,7354.443623 +0.084668,2710,7361.760124 +-0.127489,2255,7370.328627 +0.011139,2276,7377.624908 +0.200432,6621,7397.793312 +0.512662,1631,7403.37194 +0.350538,2727,7412.132349 +0.17123,2034,7418.292473 +0.056484,2350,7425.38721 +-0.047422,1429,7429.624414 +0.058959,1903,7436.474757 +-0.130189,1255,7440.379282 +0.25322,1468,7446.113152 +0.058484,1427,7450.228447 +-0.072675,2059,7456.305244 +-0.139121,1593,7461.770935 +-0.144111,1549,7465.595454 +0.095023,1913,7471.528526 +-0.158304,1740,7477.192059 +-0.076274,1547,7481.303122 +0.072728,1652,7487.459523 +-0.055531,2317,7494.864194 +-0.133189,1557,7500.672775 +-0.35,1349,7504.76235 +0.410069,2536,7513.770541 +0.261378,1300,7517.380375 +0.108302,1309,7520.896391 +0.32158,2354,7527.602602 +0.015855,1217,7532.682105 +0.246053,1958,7537.16981 +0.288598,1784,7543.24433 +-0.109708,704,7546.147681 +-0.057531,1190,7549.766725 +-0.118312,684,7552.473243 +0.34894,2661,7560.312189 +0.047136,951,7563.593174 +0.25122,1426,7569.417938 +0.209936,1229,7572.80815 +0.144711,1515,7576.603079 +0.079538,2018,7582.712487 +-0.108121,1135,7586.016139 +0.186866,1101,7589.156592 +0.306404,2336,7597.462422 +-0.077591,1419,7601.106096 +0.708106,1000,7604.441884 +0.08487,806,7607.3616 +-0.058913,1170,7610.942386 +0.545498,2065,7617.722328 +0.079028,933,7621.004918 +0.181167,1648,7627.113026 +0.135564,1625,7631.367309 +0.378571,2432,7639.725504 +-0.022623,1210,7643.100652 +0.343808,2505,7649.989471 +0.515078,1833,7655.788805 +0.497518,1883,7661.915326 +0.447311,1768,7667.960258 +0.658211,1149,7671.319555 +0.53464,922,7674.506827 +-0.206389,636,7675.551201 +0.433769,2562,7684.741263 +0.425258,2150,7691.491993 +0.502401,1670,7695.467297 +0.495924,1365,7700.634034 +-0.067307,1146,7703.889017 +0.440007,2273,7710.440969 +-0.054531,1036,7713.543747 +-0.001357,1278,7716.936149 +0.204396,1563,7722.533118 +-0.05906,1478,7726.726277 +0.018225,1524,7732.554766 +-0.032295,1194,7736.346183 +0.013225,1182,7740.021631 +-0.015623,1875,7746.449881 +-0.006771,1582,7750.774651 +-0.10733,1350,7755.990352 +0.06587,1896,7761.956896 +0.083538,2104,7768.282504 +0.004139,1376,7771.931926 +-0.117111,1147,7775.204176 +-0.05806,1158,7778.501034 +0.023518,2194,7785.637445 +0.051484,1608,7791.68095 +0.183866,2464,7799.553043 +-0.023406,1567,7805.429028 +-0.065133,1337,7809.536716 +0.053959,1706,7815.334217 +0.233692,1646,7820.852117 +0.534401,1757,7824.992982 +0.164515,1914,7830.859568 +0.370113,2990,7839.956648 +0.413189,2224,7847.056374 +0.161322,1923,7853.340841 +0.306734,1485,7859.098086 +0.324621,2472,7866.756829 +0.137519,2591,7874.230381 +0.41523,1853,7880.250185 +0.463541,2132,7886.78198 +-0.003357,1260,7891.960317 +0.35397,1501,7895.784774 +0.476808,1985,7901.971638 +0.470541,1894,7908.461114 +0.267527,2032,7914.970028 +0.359955,1959,7921.549664 +0.367876,2935,7931.495661 +0.358557,3166,7940.975351 +0.480572,2379,7947.641581 +-0.078913,1868,7953.591977 +0.410962,3653,7964.56509 +0.045971,2515,7973.834343 +0.596147,1407,7978.048554 +0.531273,2164,7984.986622 +0.328258,2720,7994.094134 +0.680593,1078,7997.371024 +0.112028,1726,8003.061945 +0.232924,2077,8009.333768 +0.502541,2608,8016.310991 +0.238688,2130,8022.748544 +0.314206,1934,8029.20437 +0.44481,1937,8035.558279 +0.159495,1861,8041.818151 +0.338773,2838,8051.303128 +0.331098,1922,8057.398742 +-0.020531,829,8058.594579 +-0.131312,874,8061.487954 +0.52718,1349,8066.631847 +-0.33,724,8067.693624 +0.381905,1742,8073.359874 +0.163701,1782,8079.123339 +0.456697,1658,8083.386266 +0.499251,1377,8088.706672 +0.172124,1904,8095.206384 +-0.321,560,8096.134335 +0.464888,2184,8102.962576 +0.237965,1919,8109.403981 +0.057421,1183,8112.763895 +0.238965,1783,8118.678681 +0.386462,2923,8127.725682 +-0.058531,2083,8134.137103 +0.01055,1904,8140.136525 +0.093926,2366,8147.1555 +0.349943,3654,8159.899681 +0.364663,3840,8170.989666 +0.290098,2930,8179.991883 +0.321621,2830,8188.844911 +-0.05406,11559,8225.918204 +0.603443,1107,8229.649346 +-0.301704,810,8232.527807 +0.724379,786,8235.294538 +0.203984,1237,8238.810967 +-0.029406,1370,8244.228987 +-0.341,778,8245.370353 +-0.055531,1085,8248.573262 +-0.202034,936,8251.580722 +0.2783,1649,8257.332073 +0.130807,1417,8261.13381 +0.503515,1529,8266.93986 +0.185432,1342,8270.850161 +0.259527,1679,8276.879414 +-0.072307,1538,8281.176495 +-0.055674,1078,8284.563461 +-0.007138,687,8287.513818 +0.544778,1200,8291.040457 +0.134043,923,8293.924237 +0.090495,903,8296.812253 +0.054484,1088,8299.980766 +0.327699,1439,8303.649385 +0.643709,912,8306.646952 +0.050484,737,8309.294598 +0.506468,1481,8313.087323 +0.240488,919,8315.994892 +0.25422,1119,8319.171668 +0.257453,1180,8324.532991 +-0.302704,811,8325.943463 +0.422094,2097,8333.053019 +0.539215,1483,8338.894663 +0.25122,2493,8346.61376 +0.174396,1810,8352.837762 +0.717612,927,8355.947262 +0.224053,2098,8362.270691 +0.329699,1849,8368.495738 +0.299734,2517,8375.830219 +0.188157,2438,8384.493001 +0.460652,1677,8388.89849 +0.126519,1801,8394.893211 +0.421109,2006,8401.531183 +0.197014,1875,8407.728225 +0.213936,2019,8414.495124 +0.35197,1476,8418.265205 +0.10465,1867,8424.231416 +0.591953,1243,8427.776566 +0.275863,2188,8434.240082 +0.417483,2161,8442.946673 +0.628528,1214,8446.605994 +0.257495,1993,8453.119108 +0.130807,2155,8459.968418 +0.193866,1545,8464.27452 +0.161322,989,8467.584927 +0.649726,1039,8471.042084 +0.542644,1271,8476.231449 +0.211936,1685,8480.332281 +0.546629,1893,8486.307214 +0.479452,1895,8492.474297 +0.144495,1615,8496.669073 +0.184396,1689,8502.519696 +0.199564,1523,8508.337996 +0.259783,1755,8512.957785 +0.205564,1867,8519.519014 +0.283984,2112,8526.337322 +0.147495,1380,8532.107956 +0.348737,1942,8536.640707 +0.242053,2025,8543.054471 +0.127564,2004,8549.139486 +-0.025295,1617,8554.637595 +0.349908,2094,8561.074543 +-0.086133,1553,8565.472458 +-0.339,914,8568.83683 +0.114519,1826,8575.231667 +0.06787,1849,8581.676151 +-0.16244,1157,8585.454049 +0.553786,1257,8589.422538 +-0.326,795,8592.277899 +0.582831,1494,8597.657163 +0.020881,2515,8604.613717 +0.270667,2478,8611.518872 +-0.106133,980,8614.673984 +0.547204,1414,8619.936675 +0.471542,1952,8626.644103 +0.493786,1880,8633.224904 +-0.004357,1231,8636.870582 +0.2513,1757,8641.54044 +0.281984,1937,8648.088308 +0.023518,1952,8654.170644 +0.492187,1672,8659.861657 +0.383541,2608,8666.809605 +0.234876,2179,8674.806142 +0.348,1836,8679.083841 +0.026136,2828,8689.138344 +0.605413,1079,8692.619756 +0.182714,1761,8698.957198 +0.171124,1137,8702.54288 +-0.351,765,8705.59548 +0.335808,2219,8712.344451 +0.181883,2100,8718.583604 +0.115519,2059,8724.814011 +0.321538,1750,8730.734331 +0.338661,1543,8734.707125 +0.416575,2414,8743.285412 +0.34845,1446,8747.150629 +0.172059,2449,8754.378422 +0.211053,2726,8763.510882 +-0.037295,2243,8770.185641 +0.244138,2556,8778.795629 +0.239783,2397,8785.534515 +0.24922,1740,8791.403489 +0.506152,1532,8795.226704 +0.260167,2310,8801.964056 +0.129314,2211,8812.433811 +0.406917,2687,8820.630056 +0.478317,1451,8826.725393 +-0.032282,2511,8834.220695 +0.469924,2938,8844.261599 +0.147711,2435,8851.760725 +0.408414,1873,8858.232635 +0.256453,2547,8867.82813 +0.286053,1331,8871.926818 +0.677024,882,8875.137198 +0.35097,3352,8886.021838 +0.109948,1698,8892.375193 +0.119519,1793,8898.687383 +0.071728,1779,8903.264689 +0.613803,1704,8909.470993 +0.399355,2655,8918.834705 +0.476413,1221,8922.514926 +0.342843,1987,8929.583241 +0.640779,1225,8933.491396 +0.390733,3160,8944.188872 +0.488396,2472,8952.061294 +0.347908,2544,8961.281587 +0.385776,2325,8968.468217 +0.505427,1538,8974.333568 +0.294206,2628,8981.988766 +0.175059,1428,8987.722642 +-0.079591,1610,8992.030082 +0.286042,1447,8997.917967 +-0.075307,1544,9002.104958 +0.099028,1792,9008.533449 +0.507176,1248,9012.271259 +0.30845,2891,9022.162931 +0.112703,1008,9025.470801 +-0.05885,1123,9028.955804 +0.420321,2222,9035.914469 +0.280799,2364,9043.188725 +0.383811,2483,9052.792629 +-0.174719,1659,9057.287931 +-0.025295,1109,9062.613318 +-0.34,707,9063.808647 +-0.276452,1194,9067.611562 +0.33158,2339,9076.513098 +0.103344,1380,9080.365199 +-0.014771,1378,9084.255263 +-0.036295,1550,9090.209013 +-0.34,900,9093.348933 +0.065421,1592,9097.578418 +-0.042298,800,9100.55485 +-0.058531,1762,9106.767682 +0.098305,996,9110.020326 +0.138272,1382,9113.900784 +-0.080274,1218,9117.574631 +-0.101133,1172,9121.204749 +-0.137189,1398,9126.929999 +-0.022406,1132,9130.498577 +-0.344,1392,9134.381 +0.154921,1736,9140.591604 +0.115948,1958,9147.099252 +-0.025337,1794,9153.484286 +0.146921,2080,9160.152147 +0.014881,1254,9164.110757 +0.268527,1784,9170.84759 +-0.016332,1597,9175.403139 +-0.116,1600,9181.687542 +0.069421,1430,9186.081166 +-0.000406,777,9189.259108 +0.091023,1079,9192.838141 +0.695665,879,9195.979869 +0.539001,1423,9200.037938 +0.633109,1242,9204.101888 +0.535865,1263,9209.722444 +0.184714,1385,9213.714225 +0.211053,1297,9217.505085 +0.06187,905,9220.945875 +0.128519,1511,9227.183677 +0.252138,1390,9231.4105 +0.622848,1212,9235.378855 +0.095368,781,9238.518782 +0.187557,937,9241.778562 +0.586692,1164,9245.560947 +-0.327,602,9248.298558 +0.21258,1213,9252.224421 +0.599183,1294,9255.956561 +-0.177791,947,9259.121361 +-0.139189,681,9261.945591 +0.137043,912,9265.156599 +0.148711,935,9266.619707 +0.114396,1106,9271.92098 +0.065994,935,9273.405191 +0.364843,1559,9279.282578 +0.067959,846,9282.381462 +-0.138189,560,9283.350982 +-0.228113,771,9286.519524 +-0.058531,715,9289.523735 +0.133807,1008,9293.160452 +0.219277,1227,9297.103878 +-0.249923,1684,9303.617581 +-0.275322,1057,9306.95552 +-0.351,886,9310.198537 +-0.081603,1302,9313.810947 +-0.351,900,9316.827981 +0.437326,2135,9323.297344 +0.44337,2177,9329.868601 +0.2903,1963,9337.323388 +0.344908,2270,9344.731814 +-0.097603,1460,9348.781978 +-0.128312,1741,9355.078686 +0.187014,1143,9358.634718 +0.125792,1263,9362.347845 +0.17023,2491,9371.53837 +0.196157,2347,9378.654622 +0.204692,1498,9384.383464 +0.453452,1895,9390.926672 +0.281799,2845,9398.893615 +0.402094,1566,9404.958967 +-0.014623,1264,9408.994035 +-0.053531,1211,9412.756188 +-0.18806,1265,9416.704011 +-0.017406,846,9419.802929 +-0.141111,1020,9423.24905 +-0.005563,1336,9429.043645 +0.01555,1706,9433.467349 +-0.178791,1379,9439.055954 +0.124519,1450,9443.037458 +0.004663,988,9446.276842 +-0.011771,1493,9450.351884 +0.076139,1350,9455.93089 +0.094368,1536,9460.066689 +0.113948,1776,9466.383154 +-0.270704,857,9469.48643 +0.065959,1536,9473.65579 +0.528152,1532,9479.538274 +-0.035422,930,9482.770559 +0.028136,1088,9486.269268 +-0.249791,927,9489.445099 +0.315,1699,9493.873626 +-0.260322,1137,9499.080485 +-0.178268,928,9500.558508 +0.138692,1514,9506.42015 +0.095668,1654,9512.489796 +0.346737,1585,9516.800315 +0.27822,1501,9522.785475 +0.5472,1155,9526.565085 +0.44691,1534,9530.914695 +0.557215,1483,9536.932656 +0.434508,1465,9541.271383 +0.058437,1429,9545.526352 +0.447791,1612,9551.77328 +0.324042,1702,9558.054865 +0.40716,2255,9565.183127 +0.499139,2404,9572.512318 +0.293153,2057,9579.348116 +0.333341,2192,9586.764838 +0.31545,2051,9594.015859 +0.55202,1731,9600.621291 +0.095023,2198,9607.942578 +0.663778,1200,9611.916975 +0.02055,1632,9618.296708 +0.440234,1875,9624.44946 +0.426777,1639,9629.278634 +0.283984,2024,9637.483425 +0.413804,1743,9645.405545 +0.345808,2085,9653.713054 +0.697528,1171,9657.456864 +-0.345,1068,9661.693827 +0.328308,1700,9669.01629 +-0.15344,1588,9676.622054 +0.495501,1756,9682.320488 +-0.161979,1176,9685.742069 +0.238876,1979,9692.253916 +-0.338,822,9695.262573 +0.321538,2469,9704.072829 +-0.142708,1614,9708.190741 +0.357737,2257,9715.061541 +0.03132,1490,9720.929062 +-0.281311,1685,9725.182852 +0.233783,1880,9731.279319 +-0.129312,1163,9734.64942 +0.335661,2309,9742.98312 +-0.193301,733,9744.126235 +0.146042,1392,9750.04841 +-0.136121,1048,9754.096798 +-0.345,1237,9758.905269 +0.176396,1728,9766.727719 +0.228488,1507,9772.096863 +0.185714,1392,9777.204087 +0.053959,1630,9784.584174 +0.090437,862,9788.443965 +-0.296704,1282,9792.834872 +-0.119312,683,9795.404369 +0.03432,1110,9798.657723 +0.204432,1012,9801.761656 +0.163124,1978,9808.425494 +0.050994,1434,9813.323866 +0.162883,1215,9817.987822 +0.011225,1830,9825.921638 +0.219053,1720,9833.577734 +0.09187,1699,9841.476802 +-0.069118,762,9843.097113 +0.003855,1478,9850.427651 +0.077139,1420,9855.642607 +0.162948,1355,9860.861471 +-0.06106,1792,9868.367412 +0.432716,2326,9875.015212 +-0.009771,936,9877.942922 +0.511512,1554,9883.724778 +0.599472,1035,9887.213377 +0.476603,1745,9893.608616 +0.10465,1477,9897.810396 +0.281863,1300,9901.70241 +0.187714,873,9905.139867 +0.107948,1479,9912.431255 +0.191297,1523,9917.753748 +-0.348,879,9921.66522 +0.31858,1346,9926.534055 +-0.35,616,9929.852669 +-0.349,567,9931.138176 +0.141495,1146,9937.756367 +-0.127312,848,9939.583571 +0.619711,1251,9946.225687 +0.271453,1556,9951.575055 +0.356908,1686,9959.27784 +0.331661,1266,9964.014715 +0.052519,1265,9968.794289 +0.336773,1953,9975.054784 +-0.333,837,9977.938066 +0.436685,2294,9984.548091 +0.175557,1814,9990.524363 +-0.352,1725,9996.197741 +-0.277416,1667,10000.33215 +-0.35,1447,10006.903015 +0.025136,1643,10012.386861 +-0.351,1525,10019.757132 +-0.343,1344,10024.833764 +0.046994,1989,10033.252281 +0.35497,2603,10044.794616 +-0.136121,1829,10050.613914 +-0.055531,1523,10057.826907 +-0.067307,1374,10062.718791 +0.002402,2429,10074.011074 +-0.332,1519,10079.206482 +-0.168677,1281,10083.94434 +0.014225,1335,10090.812446 +-0.079675,1903,10096.920312 +-0.06685,1805,10104.158103 +-0.300704,1561,10109.578006 +-0.172979,1469,10113.228468 +0.034737,2008,10119.274612 +0.646045,1115,10122.7493 +0.262598,1848,10129.164333 +0.478335,1841,10135.587393 +0.536683,1561,10139.908014 +-0.163677,726,10142.769332 +0.40535,2418,10150.081842 +0.259453,2021,10156.558774 +-0.024502,1262,10161.650701 +-0.042282,930,10164.707554 +-0.049422,765,10165.845477 +0.285984,1404,10171.378414 +0.478016,1438,10175.118942 +-0.313,766,10178.008039 +0.084302,1097,10187.005437 +0.345538,2239,10194.005328 +0.01955,909,10197.09613 +0.046994,1351,10201.01025 +0.454276,1897,10207.311392 +0.229014,1328,10210.97305 +0.31958,1703,10216.841009 +0.57048,1019,10220.015394 +0.01055,1076,10223.267362 +0.454079,1673,10229.224617 +0.426439,2176,10235.881055 +0.504598,2025,10242.103011 +0.681846,1033,10245.416895 +0.290153,2290,10252.698996 +0.2853,1265,10256.500966 +0.359876,2733,10265.964206 +0.07187,1951,10272.560488 +0.621971,1712,10278.407147 +-0.161677,2194,10285.101516 +-0.001357,1902,10291.125623 +0.342876,2593,10298.111219 +0.35497,3606,10310.776561 +0.193124,1386,10314.76463 +0.31645,3071,10324.817897 +0.489152,2018,10331.390096 +0.072728,1138,10334.66247 +-0.352,579,10337.176966 +-0.334,641,10338.189691 +0.299863,1493,10343.754897 +-0.345,581,10344.642584 +0.433352,1320,10350.075285 +-0.342,801,10351.285367 +0.34994,1299,10356.509296 +0.127564,1273,10360.113587 +0.412065,1791,10366.618155 +0.24645,1354,10370.621154 +0.35097,1387,10374.629982 +-0.138677,870,10377.735433 +0.562832,1231,10381.300071 +0.022881,1333,10386.794741 +-0.083913,1163,10390.323088 +0.007139,927,10393.45161 +0.039518,665,10394.463485 +0.307308,999,10397.754746 +-0.123719,638,10400.412601 +0.484186,1591,10404.46817 +0.569722,1002,10407.661551 +0.134043,1557,10413.324012 +-0.134312,545,10415.781434 +0.156322,1121,10419.204554 +0.01455,979,10422.382061 +0.066437,924,10423.817133 +-0.002357,796,10426.622307 +0.264527,1252,10430.354993 +0.584003,1121,10435.470129 +0.453838,2260,10442.383041 +0.189014,2199,10449.152822 +0.54835,1652,10454.854498 +0.186816,3044,10464.207938 +0.610485,1473,10468.065901 +0.190866,2543,10475.032645 +0.274863,1978,10481.529024 +-0.00313,2312,10490.405779 +0.253053,3101,10500.643087 +0.086668,1320,10504.452186 +0.127057,2842,10513.918492 +-0.125489,1423,10517.857187 +0.064518,1640,10523.572568 +0.270734,2602,10530.794524 +0.323495,2244,10537.276643 +0.206816,1583,10542.842922 +0.41763,2378,10550.11211 +-0.35,606,10552.69151 +0.022518,1068,10555.988159 +0.326,1634,10561.652667 +0.583874,1151,10565.08183 +0.220384,1877,10569.635858 +0.081538,2719,10578.877268 +0.157124,1615,10584.58867 +0.144711,1845,10590.556854 +-0.334,663,10591.575933 +-0.130312,1057,10594.887073 +0.193297,1386,10600.366652 +0.128564,2321,10607.725653 +0.401928,2621,10615.475762 +0.131807,1689,10621.783959 +0.244589,1183,10625.438225 +0.520613,1353,10631.057469 +0.442329,1778,10635.187839 +0.220277,1299,10640.34815 +-0.058913,1052,10643.551044 +0.098028,1568,10647.635385 +-0.15335,1111,10650.961933 +0.129792,1253,10654.429516 +-0.004138,1333,10659.690359 +-0.352,1417,10663.713793 +0.154124,1553,10669.594352 +0.149272,1851,10674.349936 +0.209692,1967,10680.893208 +0.073139,2017,10687.633584 +-0.347,844,10690.511572 +-0.189304,1169,10693.899713 +-0.163979,1521,10699.355731 +0.004855,2441,10706.058564 +-0.226557,1412,10711.258245 +0.227167,1891,10715.588298 +-0.034345,1272,10720.606355 +-0.082913,1419,10724.831479 +0.015881,1417,10728.992351 +0.072959,1328,10734.525931 +0.289042,1573,10739.018857 +0.315495,2413,10748.048215 +0.443156,1663,10752.562618 +0.286042,1790,10758.726717 +0.31145,2105,10765.544843 +-0.115312,1065,10768.829353 +0.588874,1151,10772.312763 +0.422452,1895,10778.390475 +0.200564,1803,10784.553664 +0.022518,1441,10788.536764 +0.305404,2135,10795.176641 +0.291098,1712,10801.079039 +-0.305704,1283,10804.730947 +-0.14944,947,10807.776403 +0.280734,2066,10814.203199 +-0.069274,1327,10817.920161 +0.744418,812,10820.756027 +0.323404,1689,10826.589612 +0.677377,904,10829.528356 +-0.006942,1464,10833.373283 +-0.144121,1662,10839.13562 +0.431998,2289,10846.012632 +0.037437,1729,10851.974735 +0.287598,1575,10856.452862 +0.018792,1290,10862.043998 +0.524103,1801,10866.565504 +0.710544,861,10869.647034 +0.166322,1677,10875.551189 +0.076538,1364,10879.345465 +0.427678,2247,10885.972412 +0.296098,1552,10891.505085 +0.31545,2158,10898.003943 +-0.290452,1631,10902.211606 +-0.112489,1278,10907.556134 +0.389786,1880,10913.81234 +0.285378,1240,10917.258223 +0.196157,1847,10923.792352 +-0.034295,1376,10927.547058 +0.146495,1880,10933.650134 +0.399362,2677,10940.818152 +-0.113,1163,10945.882157 diff --git a/004_image_stack_ram_based_reward_custom/street_fighter_custom_wrapper.py b/004_image_stack_ram_based_reward_custom/street_fighter_custom_wrapper.py index 3b3e622..b1f8ac3 100644 --- a/004_image_stack_ram_based_reward_custom/street_fighter_custom_wrapper.py +++ b/004_image_stack_ram_based_reward_custom/street_fighter_custom_wrapper.py @@ -1,3 +1,4 @@ +import math import collections import gym @@ -13,7 +14,9 @@ class StreetFighterCustomWrapper(gym.Wrapper): self.num_frames = 3 self.frame_stack = collections.deque(maxlen=self.num_frames) - self.reward_coeff = 3 + self.reward_coeff = 3.0 + + self.total_timesteps = 0 self.full_hp = 176 self.prev_player_health = self.full_hp @@ -37,6 +40,8 @@ class StreetFighterCustomWrapper(gym.Wrapper): observation = self.env.reset() self.prev_player_health = self.full_hp self.prev_oppont_health = self.full_hp + + self.total_timesteps = 0 # Clear the frame stack and add the first observation [num_frames] times self.frame_stack.clear() @@ -50,20 +55,24 @@ class StreetFighterCustomWrapper(gym.Wrapper): obs, _reward, _done, info = self.env.step(action) curr_player_health = info['health'] curr_oppont_health = info['enemy_health'] + + self.total_timesteps += 1 # Game is over and player loses. if curr_player_health < 0: - custom_reward = -curr_oppont_health # Use the remaining health points of opponent as penalty. + custom_reward = -math.pow(self.full_hp, (curr_oppont_health + 1) / (self.full_hp + 1)) # Use the remaining health points of opponent as penalty. # If the opponent also has negative health points, it's a even game and the reward is +1. custom_done = True # Game is over and player wins. elif curr_oppont_health < 0: - custom_reward = curr_player_health * self.reward_coeff # Use the remaining health points of player as reward. + # custom_reward = curr_player_health * self.reward_coeff # Use the remaining health points of player as reward. # Multiply by reward_coeff to make the reward larger than the penalty to avoid cowardice of agent. + + custom_reward = math.pow(self.full_hp, (5940 - self.total_timesteps) / 5940) * self.reward_coeff # Use the remaining time steps as reward. custom_done = True - # While the fighting is still going on. + # While the fighting is still going on else: custom_reward = self.reward_coeff * (self.prev_oppont_health - curr_oppont_health) - (self.prev_player_health - curr_player_health) self.prev_player_health = curr_player_health @@ -75,5 +84,5 @@ class StreetFighterCustomWrapper(gym.Wrapper): custom_done = False # Max reward is 6 * full_hp = 1054 (damage * 3 + winning_reward * 3) - return self._preprocess_observation(obs), custom_reward, custom_done, info + return self._preprocess_observation(obs), 0.001 * custom_reward, custom_done, info # reward normalization \ No newline at end of file diff --git a/004_image_stack_ram_based_reward_custom/test.py b/004_image_stack_ram_based_reward_custom/test.py index 7ad01a4..06df259 100644 --- a/004_image_stack_ram_based_reward_custom/test.py +++ b/004_image_stack_ram_based_reward_custom/test.py @@ -20,30 +20,44 @@ def make_env(game, state): game = "StreetFighterIISpecialChampionEdition-Genesis" state_stages = [ "Champion.Level1.RyuVsGuile", - "Champion.Level1.ChunLiVsGuile", # Average reward for random strategy: -102.3 | -20.4 - "ChampionX.Level1.ChunLiVsKen", # Average reward for random strategy: -247.6 - "Champion.Level2.ChunLiVsKen", - "Champion.Level3.ChunLiVsChunLi", - "Champion.Level4.ChunLiVsZangief", - "Champion.Level5.ChunLiVsDhalsim", - "Champion.Level6.ChunLiVsRyu", - "Champion.Level7.ChunLiVsEHonda", - "Champion.Level8.ChunLiVsBlanka", - "Champion.Level9.ChunLiVsBalrog", - "Champion.Level10.ChunLiVsVega", - "Champion.Level11.ChunLiVsSagat", - "Champion.Level12.ChunLiVsBison" - # Add other stages as necessary + "Champion.Level2.RyuVsKen", + "Champion.Level3.RyuVsChunLi", + "Champion.Level4.RyuVsZangief", + "Champion.Level5.RyuVsDhalsim", + "Champion.Level6.RyuVsRyu", + "Champion.Level7.RyuVsEHonda", + "Champion.Level8.RyuVsBlanka", + "Champion.Level9.RyuVsBalrog", + "Champion.Level10.RyuVsVega", + "Champion.Level11.RyuVsSagat", + "Champion.Level12.RyuVsBison" ] +# state_stages = [ +# "Champion.Level1.RyuVsGuile", +# "Champion.Level1.ChunLiVsGuile", # Average reward for random strategy: -102.3 | -20.4 +# "ChampionX.Level1.ChunLiVsKen", # Average reward for random strategy: -247.6 +# "Champion.Level2.ChunLiVsKen", +# "Champion.Level3.ChunLiVsChunLi", +# "Champion.Level4.ChunLiVsZangief", +# "Champion.Level5.ChunLiVsDhalsim", +# "Champion.Level6.ChunLiVsRyu", +# "Champion.Level7.ChunLiVsEHonda", +# "Champion.Level8.ChunLiVsBlanka", +# "Champion.Level9.ChunLiVsBalrog", +# "Champion.Level10.ChunLiVsVega", +# "Champion.Level11.ChunLiVsSagat", +# "Champion.Level12.ChunLiVsBison" +# # Add other stages as necessary +# ] -env = make_env(game, state_stages[0])() +env = make_env(game, state_stages[11])() model = PPO( "CnnPolicy", env, verbose=1 ) -model_path = r"trained_models_level_1/ppo_ryu_000000_steps" +model_path = r"trained_models_ryu_level_1_time_reward_small_random/ppo_ryu_2600000_steps" model.load(model_path) # Average reward for optuna/trial_1_best_model: -82.3 # Average reward for optuna/trial_9_best_model: 36.7 | -86.23 @@ -60,6 +74,7 @@ for _ in range(num_episodes): obs = env.reset() total_reward = 0 while not done: + # while True: timestamp = time.time() action, _states = model.predict(obs) obs, reward, done, info = env.step(action) @@ -68,9 +83,9 @@ for _ in range(num_episodes): total_reward += reward print("Reward: {}, playerHP: {}, enemyHP:{}".format(reward, info['health'], info['enemy_health'])) env.render() - time.sleep(0.01) + # time.sleep(0.005) print("Total reward: {}".format(total_reward)) episode_reward_sum += total_reward -env.close() -print("Average reward for {}: {}".format(model_path, episode_reward_sum/num_episodes)) \ No newline at end of file +# env.close() +# print("Average reward for {}: {}".format(model_path, episode_reward_sum/num_episodes)) \ No newline at end of file diff --git a/004_image_stack_ram_based_reward_custom/train.py b/004_image_stack_ram_based_reward_custom/train.py index 510861e..fa06234 100644 --- a/004_image_stack_ram_based_reward_custom/train.py +++ b/004_image_stack_ram_based_reward_custom/train.py @@ -1,4 +1,5 @@ import os +import sys import random import retro @@ -8,7 +9,7 @@ from stable_baselines3.common.callbacks import BaseCallback, CheckpointCallback from street_fighter_custom_wrapper import StreetFighterCustomWrapper -LOG_DIR = 'logs/' +LOG_DIR = 'logs' os.makedirs(LOG_DIR, exist_ok=True) class RandomOpponentChangeCallback(BaseCallback): @@ -58,21 +59,36 @@ def main(): state_stages = [ "Champion.Level1.RyuVsGuile", - "Champion.Level1.ChunLiVsGuile", # Average reward for random strategy: -102.3 - "Champion.Level2.ChunLiVsKen", - "Champion.Level3.ChunLiVsChunLi", - "Champion.Level4.ChunLiVsZangief", - "Champion.Level5.ChunLiVsDhalsim", - "Champion.Level6.ChunLiVsRyu", - "Champion.Level7.ChunLiVsEHonda", - "Champion.Level8.ChunLiVsBlanka", - "Champion.Level9.ChunLiVsBalrog", - "Champion.Level10.ChunLiVsVega", - "Champion.Level11.ChunLiVsSagat", - "Champion.Level12.ChunLiVsBison" - # Add other stages as necessary + "Champion.Level2.RyuVsKen", + "Champion.Level3.RyuVsChunLi", + "Champion.Level4.RyuVsZangief", + "Champion.Level5.RyuVsDhalsim", + "Champion.Level6.RyuVsRyu", + "Champion.Level7.RyuVsEHonda", + "Champion.Level8.RyuVsBlanka", + "Champion.Level9.RyuVsBalrog", + "Champion.Level10.RyuVsVega", + "Champion.Level11.RyuVsSagat", + "Champion.Level12.RyuVsBison" ] + # state_stages = [ + # "Champion.Level1.RyuVsGuile", + # "Champion.Level1.ChunLiVsGuile", # Average reward for random strategy: -102.3 + # "Champion.Level2.ChunLiVsKen", + # "Champion.Level3.ChunLiVsChunLi", + # "Champion.Level4.ChunLiVsZangief", + # "Champion.Level5.ChunLiVsDhalsim", + # "Champion.Level6.ChunLiVsRyu", + # "Champion.Level7.ChunLiVsEHonda", + # "Champion.Level8.ChunLiVsBlanka", + # "Champion.Level9.ChunLiVsBalrog", + # "Champion.Level10.ChunLiVsVega", + # "Champion.Level11.ChunLiVsSagat", + # "Champion.Level12.ChunLiVsBison" + # # Add other stages as necessary + # ] + # state_stages = [ # "ChampionX.Level1.ChunLiVsKen", # Average reward for random strategy: -247.6 # "ChampionX.Level2.ChunLiVsChunLi", @@ -103,31 +119,27 @@ def main(): n_steps=1024, batch_size=64, learning_rate=1e-4, - ent_coef=0.01, - clip_range=0.2, - gamma=0.95, - gae_lambda=0.81322, - tensorboard_log="logs/" + tensorboard_log="logs" ) # Set the save directory - save_dir = "trained_models_ryu_level_1_reward_x3" + save_dir = "trained_models_ryu_level_1_time_reward_small_random" os.makedirs(save_dir, exist_ok=True) # Load the model from file - # model_path = "trained_models/ppo_chunli_1296000_steps.zip" + model_path = "trained_models_ryu_level_1_time_reward_small_continue/ppo_ryu_400000_steps.zip" # Load model and modify the learning rate and entropy coefficient # custom_objects = { # "learning_rate": 0.0002 # } - # model = PPO.load(model_path, env=env, device="cuda")#, custom_objects=custom_objects) + model = PPO.load(model_path, env=env, device="cuda")#, custom_objects=custom_objects) # Set up callbacks - # opponent_interval = 35840 # stage_interval * num_envs = total_steps_per_stage + opponent_interval = 32768 # stage_interval * num_envs = total_steps_per_stage checkpoint_interval = 200000 # checkpoint_interval * num_envs = total_steps_per_checkpoint (Every 80 rounds) checkpoint_callback = CheckpointCallback(save_freq=checkpoint_interval, save_path=save_dir, name_prefix="ppo_ryu") - # stage_increase_callback = RandomOpponentChangeCallback(state_stages, opponent_interval, save_dir) + stage_increase_callback = RandomOpponentChangeCallback(state_stages, opponent_interval, save_dir) # model_params = { # 'n_steps': 5, @@ -141,12 +153,21 @@ def main(): # } # model = A2C('CnnPolicy', env, tensorboard_log='logs/', verbose=1, **model_params, policy_kwargs=dict(optimizer_class=RMSpropTF)) - model.learn( - total_timesteps=int(10000000), # total_timesteps = stage_interval * num_envs * num_stages (1120 rounds) - callback=[checkpoint_callback]#, stage_increase_callback] - ) - env.close() + # Writing the training logs from stdout to a file + original_stdout = sys.stdout + log_file_path = os.path.join(save_dir, "training_log.txt") + with open(log_file_path, 'w') as log_file: + sys.stdout = log_file + + model.learn( + total_timesteps=int(10000000), # total_timesteps = stage_interval * num_envs * num_stages (1120 rounds) + callback=[checkpoint_callback, stage_increase_callback] + ) + env.close() + # Restore stdout + sys.stdout = original_stdout + # Save the final model model.save(os.path.join(save_dir, "ppo_sf2_ryu_final.zip")) diff --git a/004_image_stack_ram_based_reward_custom/tune_ppo.py b/004_image_stack_ram_based_reward_custom/tune_ppo.py new file mode 100644 index 0000000..f2d3184 --- /dev/null +++ b/004_image_stack_ram_based_reward_custom/tune_ppo.py @@ -0,0 +1,68 @@ +import os + +import retro +import optuna +from stable_baselines3 import PPO +from stable_baselines3.common.monitor import Monitor +from stable_baselines3.common.evaluation import evaluate_policy + +from street_fighter_custom_wrapper import StreetFighterCustomWrapper + +LOG_DIR = 'logs/' +OPT_DIR = 'optuna/' +os.makedirs(LOG_DIR, exist_ok=True) +os.makedirs(OPT_DIR, exist_ok=True) + +def optimize_ppo(trial): + return { + 'n_steps':trial.suggest_int('n_steps', 512, 2048, log=True), + 'gamma':trial.suggest_float('gamma', 0.9, 0.9999), + 'learning_rate':trial.suggest_float('learning_rate', 5e-5, 5e-4, log=True), + 'gae_lambda':trial.suggest_float('gae_lambda', 0.8, 0.9999) + } + +def make_env(game, state): + def _init(): + env = retro.make( + game=game, + state=state, + use_restricted_actions=retro.Actions.FILTERED, + obs_type=retro.Observations.IMAGE + ) + env = StreetFighterCustomWrapper(env) + return env + return _init + +def optimize_agent(trial): + game = "StreetFighterIISpecialChampionEdition-Genesis" + state = "Champion.Level1.ChunLiVsGuile"#"ChampionX.Level1.ChunLiVsKen" + + try: + model_params = optimize_ppo(trial) + + # Create environment + env = make_env(game, state)() + env = Monitor(env, LOG_DIR) + + # Create algo + model = PPO('CnnPolicy', env, verbose=1, **model_params) + model.learn(total_timesteps=500000) + + # Evaluate model + mean_reward, _ = evaluate_policy(model, env, n_eval_episodes=30, deterministic=False) + env.close() + + SAVE_PATH = os.path.join(OPT_DIR, 'trial_{}_best_model'.format(trial.number)) + model.save(SAVE_PATH) + + return mean_reward + + except Exception as e: + return -1 + +# Creating the experiment +study = optuna.create_study(direction='maximize') +study.optimize(optimize_agent, n_trials=10, n_jobs=1) + +print(study.best_params) +print(study.best_trial) diff --git a/005_rgb_stack_ram_based_reward_time_penalty/__pycache__/street_fighter_custom_wrapper.cpython-38.pyc b/005_rgb_stack_ram_based_reward_time_penalty/__pycache__/street_fighter_custom_wrapper.cpython-38.pyc new file mode 100644 index 0000000..32bba26 Binary files /dev/null and b/005_rgb_stack_ram_based_reward_time_penalty/__pycache__/street_fighter_custom_wrapper.cpython-38.pyc differ diff --git a/005_rgb_stack_ram_based_reward_time_penalty/check_reward.py b/005_rgb_stack_ram_based_reward_time_penalty/check_reward.py new file mode 100644 index 0000000..b3b7256 --- /dev/null +++ b/005_rgb_stack_ram_based_reward_time_penalty/check_reward.py @@ -0,0 +1,51 @@ +import os +import time + +import retro +from stable_baselines3.common.monitor import Monitor + +from street_fighter_custom_wrapper import StreetFighterCustomWrapper + +LOG_DIR = 'logs/' +os.makedirs(LOG_DIR, exist_ok=True) + +def make_env(game, state): + def _init(): + env = retro.make( + game=game, + state=state, + use_restricted_actions=retro.Actions.FILTERED, + obs_type=retro.Observations.IMAGE + ) + env = StreetFighterCustomWrapper(env) + return env + return _init + +game = "StreetFighterIISpecialChampionEdition-Genesis" +state = "Champion.Level1.RyuVsGuile" + +env = make_env(game, state)() +env = Monitor(env, 'logs/') + +num_episodes = 30 +episode_reward_sum = 0 +for _ in range(num_episodes): + done = False + obs = env.reset() + total_reward = 0 + while not done: + timestamp = time.time() + obs, reward, done, info = env.step(env.action_space.sample()) + + # Note that if player wins but only has 0 HP left, the winning reward is still 0, so it won't be printed. + if reward != 0: + total_reward += reward + print("Reward: {}, playerHP: {}, enemyHP:{}".format(reward, info['health'], info['enemy_health'])) + env.render() + # time.sleep(0.005) + + print("Total reward: {}".format(total_reward)) + episode_reward_sum += total_reward + +env.close() +print("Average reward for random strategy: {}".format(episode_reward_sum/num_episodes)) diff --git a/005_rgb_stack_ram_based_reward_time_penalty/custom_cnn.py b/005_rgb_stack_ram_based_reward_time_penalty/custom_cnn.py new file mode 100644 index 0000000..25c50ea --- /dev/null +++ b/005_rgb_stack_ram_based_reward_time_penalty/custom_cnn.py @@ -0,0 +1,24 @@ +import gym +import torch +import torch.nn as nn +from stable_baselines3.common.torch_layers import BaseFeaturesExtractor + +# Custom feature extractor (CNN) +class CustomCNN(BaseFeaturesExtractor): + def __init__(self, observation_space: gym.Space): + super(CustomCNN, self).__init__(observation_space, features_dim=512) + self.cnn = nn.Sequential( + nn.Conv2d(4, 32, kernel_size=5, stride=2, padding=0), + nn.ReLU(), + nn.Conv2d(32, 64, kernel_size=5, stride=2, padding=0), + nn.ReLU(), + nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=0), + nn.ReLU(), + nn.Flatten(), + nn.Linear(16384, self.features_dim), + nn.ReLU() + ) + + def forward(self, observations: torch.Tensor) -> torch.Tensor: + return self.cnn(observations) + \ No newline at end of file diff --git a/005_rgb_stack_ram_based_reward_time_penalty/evaluate.py b/005_rgb_stack_ram_based_reward_time_penalty/evaluate.py new file mode 100644 index 0000000..c435f08 --- /dev/null +++ b/005_rgb_stack_ram_based_reward_time_penalty/evaluate.py @@ -0,0 +1,52 @@ +import retro + +from stable_baselines3 import PPO +from stable_baselines3.common.vec_env import DummyVecEnv +from stable_baselines3.common.monitor import Monitor +from stable_baselines3.common.evaluation import evaluate_policy + +from custom_cnn import CustomCNN +from street_fighter_custom_wrapper import StreetFighterCustomWrapper + +def make_env(game, state): + def _init(): + env = retro.make( + game=game, + state=state, + use_restricted_actions=retro.Actions.FILTERED, + obs_type=retro.Observations.IMAGE + ) + env = StreetFighterCustomWrapper(env) + return env + return _init + +game = "StreetFighterIISpecialChampionEdition-Genesis" +state_stages = [ + "Champion.Level1.ChunLiVsGuile", + "Champion.Level2.ChunLiVsKen", + "Champion.Level3.ChunLiVsChunLi", + "Champion.Level4.ChunLiVsZangief", + "Champion.Level5.ChunLiVsDhalsim", + "Champion.Level6.ChunLiVsRyu", + "Champion.Level7.ChunLiVsEHonda", + "Champion.Level8.ChunLiVsBlanka", + "Champion.Level9.ChunLiVsBalrog", + "Champion.Level10.ChunLiVsVega", + "Champion.Level11.ChunLiVsSagat", + "Champion.Level12.ChunLiVsBison" + # Add other stages as necessary +] + +env = make_env(game, state_stages[0])() + +# Wrap the environment +# env = Monitor(env, 'logs/') + +policy_kwargs = {'features_extractor_class': CustomCNN} +model = PPO("CnnPolicy", env, policy_kwargs=policy_kwargs) + +model = PPO.load(r"dummy_model_ppo_chunli") +# model.load(r"trained_models/ppo_chunli_864000_steps") + +mean_reward, std_reward = evaluate_policy(model, env, render=True, n_eval_episodes=10, deterministic=False, return_episode_rewards=True) +print(f"Mean reward: {mean_reward:.2f} +/- {std_reward:.2f}") diff --git a/005_rgb_stack_ram_based_reward_time_penalty/logs/PPO_1/events.out.tfevents.1680427238.DESKTOP-9E17TO7.27420.0 b/005_rgb_stack_ram_based_reward_time_penalty/logs/PPO_1/events.out.tfevents.1680427238.DESKTOP-9E17TO7.27420.0 new file mode 100644 index 0000000..eaa22b1 Binary files /dev/null and b/005_rgb_stack_ram_based_reward_time_penalty/logs/PPO_1/events.out.tfevents.1680427238.DESKTOP-9E17TO7.27420.0 differ diff --git a/005_rgb_stack_ram_based_reward_time_penalty/logs/PPO_2/events.out.tfevents.1680442574.DESKTOP-9E17TO7.8472.0 b/005_rgb_stack_ram_based_reward_time_penalty/logs/PPO_2/events.out.tfevents.1680442574.DESKTOP-9E17TO7.8472.0 new file mode 100644 index 0000000..cbd9cc2 Binary files /dev/null and b/005_rgb_stack_ram_based_reward_time_penalty/logs/PPO_2/events.out.tfevents.1680442574.DESKTOP-9E17TO7.8472.0 differ diff --git a/005_rgb_stack_ram_based_reward_time_penalty/logs/PPO_3/events.out.tfevents.1680450538.DESKTOP-9E17TO7.4520.0 b/005_rgb_stack_ram_based_reward_time_penalty/logs/PPO_3/events.out.tfevents.1680450538.DESKTOP-9E17TO7.4520.0 new file mode 100644 index 0000000..611dfee Binary files /dev/null and b/005_rgb_stack_ram_based_reward_time_penalty/logs/PPO_3/events.out.tfevents.1680450538.DESKTOP-9E17TO7.4520.0 differ diff --git a/005_rgb_stack_ram_based_reward_time_penalty/logs/PPO_4/events.out.tfevents.1680493346.DESKTOP-9E17TO7.14264.0 b/005_rgb_stack_ram_based_reward_time_penalty/logs/PPO_4/events.out.tfevents.1680493346.DESKTOP-9E17TO7.14264.0 new file mode 100644 index 0000000..06955fe Binary files /dev/null and b/005_rgb_stack_ram_based_reward_time_penalty/logs/PPO_4/events.out.tfevents.1680493346.DESKTOP-9E17TO7.14264.0 differ diff --git a/005_rgb_stack_ram_based_reward_time_penalty/logs/monitor.csv b/005_rgb_stack_ram_based_reward_time_penalty/logs/monitor.csv new file mode 100644 index 0000000..19bc260 --- /dev/null +++ b/005_rgb_stack_ram_based_reward_time_penalty/logs/monitor.csv @@ -0,0 +1,72 @@ +#{"t_start": 1680493345.1918354, "env_id": null} +r,l,t +121.963397,1218,4.678451 +150.73549,1321,10.670471 +-68.875587,1685,16.915831 +110.079359,1992,23.723121 +-142.422045,2475,30.929999 +25.618847,2409,37.817851 +-10.399282,2048,44.290086 +103.210152,2378,51.321482 +-10.83336,1410,56.861604 +-109.50968,2320,63.936125 +-56.133868,1884,70.48871 +-1.263258,3856,83.55072 +137.096617,1955,89.782694 +-173.590936,2482,96.620559 +-32.06399,2622,103.507519 +-33.546645,2359,110.155689 +247.618832,631,114.254022 +34.18776,1988,121.014232 +-16.0,2372,128.411761 +142.528257,1504,131.040915 +-35.511721,2100,141.725583 +210.669563,1386,144.090119 +106.166972,1756,150.146413 +-291.452157,1771,156.139224 +-103.271598,2993,163.884984 +-3.09159,2881,176.148215 +-69.312432,2968,184.263354 +11.966888,3420,196.801994 +-72.699861,2025,203.564955 +-148.405698,1736,209.933663 +-6.505259,2863,217.596923 +-45.308495,1642,223.866472 +-60.133868,1684,230.124691 +20.320741,1912,236.920414 +12.850028,3201,245.687548 +44.331029,2020,252.592099 +4.970357,2426,260.241496 +-26.074153,3193,272.014878 +41.36973,2057,278.669485 +-58.986068,1951,285.105779 +-6.22689,3090,293.487389 +42.518735,2469,304.130861 +28.699296,1979,310.846102 +-0.123948,2276,318.259012 +-130.860871,3166,326.553931 +-94.074138,2784,337.55811 +-6.779892,1688,340.079623 +-2.846787,1842,346.163562 +-6.846787,2137,352.507397 +74.03875,2905,364.386266 +91.493381,1660,370.441151 +-60.286049,1502,372.846128 +-127.450026,2030,379.306661 +-90.028712,1565,385.293074 +-45.615818,2020,391.765668 +-49.94699,3488,403.627649 +-90.632338,2270,410.494766 +46.011777,3184,419.074744 +-11.461924,2326,429.595886 +181.774886,1260,431.709853 +-40.06399,2863,442.810548 +-90.86073,2232,449.485116 +72.549001,2173,455.957089 +66.832361,1597,461.523406 +29.003218,1923,464.330122 +-57.986068,3347,476.034669 +175.784026,1320,481.626563 +-328.0,1320,483.822258 +-81.578734,2842,495.411107 +72.161772,1818,501.982363 diff --git a/005_rgb_stack_ram_based_reward_time_penalty/street_fighter_custom_wrapper.py b/005_rgb_stack_ram_based_reward_time_penalty/street_fighter_custom_wrapper.py new file mode 100644 index 0000000..7cd7e22 --- /dev/null +++ b/005_rgb_stack_ram_based_reward_time_penalty/street_fighter_custom_wrapper.py @@ -0,0 +1,88 @@ +import math +import collections + +import gym +import numpy as np + +# Custom environment wrapper +class StreetFighterCustomWrapper(gym.Wrapper): + def __init__(self, env, testing=False): + super(StreetFighterCustomWrapper, self).__init__(env) + self.env = env + + # Use a deque to store the last 4 frames + self.num_frames = 3 + self.frame_stack = collections.deque(maxlen=self.num_frames) + + self.reward_coeff = 1.0 + + self.total_timesteps = 0 + + self.full_hp = 176 + self.prev_player_health = self.full_hp + self.prev_oppont_health = self.full_hp + + # Update observation space to include stacked grayscale images + self.observation_space = gym.spaces.Box(low=0, high=255, shape=(100, 128, 3), dtype=np.uint8) + + self.testing = testing + + def _preprocess_observation(self, observation): + + # Stack the downsampled frames. + self.frame_stack.append(observation[::2, ::2, :]) + + # Stack the R, G, B channel of each frame and return the "image". + stacked_image = np.stack([frame[:, :, i] for i, frame in enumerate(self.frame_stack)], axis=-1) + return stacked_image + + def reset(self): + observation = self.env.reset() + self.prev_player_health = self.full_hp + self.prev_oppont_health = self.full_hp + + self.total_timesteps = 0 + + # Clear the frame stack and add the first observation [num_frames] times + self.frame_stack.clear() + for _ in range(self.num_frames): + self.frame_stack.append(observation[::2, ::2, :]) + + return np.stack([frame[:, :, i] for i, frame in enumerate(self.frame_stack)], axis=-1) + + def step(self, action): + + obs, _reward, _done, info = self.env.step(action) + curr_player_health = info['health'] + curr_oppont_health = info['enemy_health'] + + self.total_timesteps += 1 + + # Game is over and player loses. + if curr_player_health < 0: + custom_reward = -math.pow(self.full_hp, (curr_oppont_health + 1) / (self.full_hp + 1)) # Use the remaining health points of opponent as penalty. + # If the opponent also has negative health points, it's a even game and the reward is +1. + custom_done = True + + # Game is over and player wins. + elif curr_oppont_health < 0: + # custom_reward = curr_player_health * self.reward_coeff # Use the remaining health points of player as reward. + # Multiply by reward_coeff to make the reward larger than the penalty to avoid cowardice of agent. + + custom_reward = math.pow(self.full_hp, (5940 - self.total_timesteps) / 5940) + custom_done = True + + # While the fighting is still going on + else: + custom_reward = self.reward_coeff * (self.prev_oppont_health - curr_oppont_health) - (self.prev_player_health - curr_player_health) + self.prev_player_health = curr_player_health + self.prev_oppont_health = curr_oppont_health + custom_done = False + + # During testing, the session should always keep going. + if self.testing: + custom_done = False + + # Max reward is 6 * full_hp = 1054 (damage * 3 + winning_reward * 3) + return self._preprocess_observation(obs), custom_reward, custom_done, info + \ No newline at end of file diff --git a/005_rgb_stack_ram_based_reward_time_penalty/test.py b/005_rgb_stack_ram_based_reward_time_penalty/test.py new file mode 100644 index 0000000..c255295 --- /dev/null +++ b/005_rgb_stack_ram_based_reward_time_penalty/test.py @@ -0,0 +1,76 @@ +import time + +import retro +from stable_baselines3 import PPO + +from street_fighter_custom_wrapper import StreetFighterCustomWrapper + +def make_env(game, state): + def _init(): + env = retro.make( + game=game, + state=state, + use_restricted_actions=retro.Actions.FILTERED, + obs_type=retro.Observations.IMAGE + ) + env = StreetFighterCustomWrapper(env) + return env + return _init + +game = "StreetFighterIISpecialChampionEdition-Genesis" +state_stages = [ + "Champion.Level1.RyuVsGuile", + "Champion.Level1.ChunLiVsGuile", # Average reward for random strategy: -102.3 | -20.4 + "ChampionX.Level1.ChunLiVsKen", # Average reward for random strategy: -247.6 + "Champion.Level2.ChunLiVsKen", + "Champion.Level3.ChunLiVsChunLi", + "Champion.Level4.ChunLiVsZangief", + "Champion.Level5.ChunLiVsDhalsim", + "Champion.Level6.ChunLiVsRyu", + "Champion.Level7.ChunLiVsEHonda", + "Champion.Level8.ChunLiVsBlanka", + "Champion.Level9.ChunLiVsBalrog", + "Champion.Level10.ChunLiVsVega", + "Champion.Level11.ChunLiVsSagat", + "Champion.Level12.ChunLiVsBison" + # Add other stages as necessary +] + +env = make_env(game, state_stages[0])() + +model = PPO( + "CnnPolicy", + env, + verbose=1 +) +model_path = r"trained_models_ryu_level_1_reward_x3/ppo_ryu_6600000_steps" +model.load(model_path) +# Average reward for optuna/trial_1_best_model: -82.3 +# Average reward for optuna/trial_9_best_model: 36.7 | -86.23 +# Average reward for trained_models/ppo_chunli_5376000_steps: -77.8 + + +obs = env.reset() +done = False + +num_episodes = 30 +episode_reward_sum = 0 +for _ in range(num_episodes): + done = False + obs = env.reset() + total_reward = 0 + while True: + timestamp = time.time() + action, _states = model.predict(obs) + obs, reward, done, info = env.step(action) + + if reward != 0: + total_reward += reward + print("Reward: {}, playerHP: {}, enemyHP:{}".format(reward, info['health'], info['enemy_health'])) + env.render() + # time.sleep(0.005) + # print("Total reward: {}".format(total_reward)) + # episode_reward_sum += total_reward + +# env.close() +# print("Average reward for {}: {}".format(model_path, episode_reward_sum/num_episodes)) \ No newline at end of file diff --git a/005_rgb_stack_ram_based_reward_time_penalty/train.py b/005_rgb_stack_ram_based_reward_time_penalty/train.py new file mode 100644 index 0000000..b17e7e5 --- /dev/null +++ b/005_rgb_stack_ram_based_reward_time_penalty/train.py @@ -0,0 +1,151 @@ +import os +import random + +import retro +from stable_baselines3 import PPO +from stable_baselines3.common.monitor import Monitor +from stable_baselines3.common.callbacks import BaseCallback, CheckpointCallback + +from street_fighter_custom_wrapper import StreetFighterCustomWrapper + +LOG_DIR = 'logs' +os.makedirs(LOG_DIR, exist_ok=True) + +class RandomOpponentChangeCallback(BaseCallback): + def __init__(self, stages, opponent_interval, verbose=0): + super(RandomOpponentChangeCallback, self).__init__(verbose) + self.stages = stages + self.opponent_interval = opponent_interval + + def _on_step(self) -> bool: + if self.n_calls % self.opponent_interval == 0: + new_state = random.choice(self.stages) + print("\nCurrent state:", new_state) + self.training_env.env_method("load_state", new_state, indices=None) + return True + +# class StageIncreaseCallback(BaseCallback): +# def __init__(self, stages, stage_interval, save_dir, verbose=0): +# super(StageIncreaseCallback, self).__init__(verbose) +# self.stages = stages +# self.stage_interval = stage_interval +# self.save_dir = save_dir +# self.current_stage = 0 + +# def _on_step(self) -> bool: +# if self.n_calls % self.stage_interval == 0 and self.current_stage < len(self.stages) - 1: +# self.current_stage += 1 +# new_state = self.stages[self.current_stage] +# self.training_env.env_method("load_state", new_state, indices=None) +# self.model.save(os.path.join(self.save_dir, f"ppo_chunli_stage_{self.current_stage}.zip")) +# return True + +def make_env(game, state): + def _init(): + env = retro.make( + game=game, + state=state, + use_restricted_actions=retro.Actions.FILTERED, + obs_type=retro.Observations.IMAGE + ) + env = StreetFighterCustomWrapper(env) + return env + return _init + +def main(): + # Set up the environment and model + game = "StreetFighterIISpecialChampionEdition-Genesis" + + state_stages = [ + "Champion.Level1.RyuVsGuile", + "Champion.Level1.ChunLiVsGuile", # Average reward for random strategy: -102.3 + "Champion.Level2.ChunLiVsKen", + "Champion.Level3.ChunLiVsChunLi", + "Champion.Level4.ChunLiVsZangief", + "Champion.Level5.ChunLiVsDhalsim", + "Champion.Level6.ChunLiVsRyu", + "Champion.Level7.ChunLiVsEHonda", + "Champion.Level8.ChunLiVsBlanka", + "Champion.Level9.ChunLiVsBalrog", + "Champion.Level10.ChunLiVsVega", + "Champion.Level11.ChunLiVsSagat", + "Champion.Level12.ChunLiVsBison" + # Add other stages as necessary + ] + + # state_stages = [ + # "ChampionX.Level1.ChunLiVsKen", # Average reward for random strategy: -247.6 + # "ChampionX.Level2.ChunLiVsChunLi", + # "ChampionX.Level3.ChunLiVsZangief", + # "ChampionX.Level4.ChunLiVsDhalsim", + # "ChampionX.Level5.ChunLiVsRyu", + # "ChampionX.Level6.ChunLiVsEHonda", + # "ChampionX.Level7.ChunLiVsBlanka", + # "ChampionX.Level8.ChunLiVsGuile", + # "ChampionX.Level9.ChunLiVsBalrog", + # "ChampionX.Level10.ChunLiVsVega", + # "ChampionX.Level11.ChunLiVsSagat", + # "ChampionX.Level12.ChunLiVsBison" + # # Add other stages as necessary + # ] + # Champion is at difficulty level 4, ChampionX is at difficulty level 8. + + env = make_env(game, state_stages[0])() + + # Warp env in Monitor wrapper to record training progress + env = Monitor(env, LOG_DIR) + + model = PPO( + "CnnPolicy", + env, + device="cuda", + verbose=1, + n_steps=2048, + batch_size=64, + learning_rate=1e-4, + gamma=0.99, + tensorboard_log="logs" + ) + + # Set the save directory + save_dir = "trained_models_ryu_level_1_time_reward" + os.makedirs(save_dir, exist_ok=True) + + # Load the model from file + # model_path = "trained_models/ppo_chunli_1296000_steps.zip" + + # Load model and modify the learning rate and entropy coefficient + # custom_objects = { + # "learning_rate": 0.0002 + # } + # model = PPO.load(model_path, env=env, device="cuda")#, custom_objects=custom_objects) + + # Set up callbacks + # opponent_interval = 35840 # stage_interval * num_envs = total_steps_per_stage + checkpoint_interval = 200000 # checkpoint_interval * num_envs = total_steps_per_checkpoint (Every 80 rounds) + checkpoint_callback = CheckpointCallback(save_freq=checkpoint_interval, save_path=save_dir, name_prefix="ppo_ryu") + # stage_increase_callback = RandomOpponentChangeCallback(state_stages, opponent_interval, save_dir) + + # model_params = { + # 'n_steps': 5, + # 'gamma': 0.99, + # 'gae_lambda':1, + # 'learning_rate': 7e-4, + # 'vf_coef': 0.5, + # 'ent_coef': 0.0, + # 'max_grad_norm':0.5, + # 'rms_prop_eps':1e-05 + # } + # model = A2C('CnnPolicy', env, tensorboard_log='logs/', verbose=1, **model_params, policy_kwargs=dict(optimizer_class=RMSpropTF)) + + model.learn( + total_timesteps=int(10000000), # total_timesteps = stage_interval * num_envs * num_stages (1120 rounds) + callback=[checkpoint_callback]#, stage_increase_callback] + ) + env.close() + + # Save the final model + model.save(os.path.join(save_dir, "ppo_sf2_ryu_final.zip")) + +if __name__ == "__main__": + main() diff --git a/005_rgb_stack_ram_based_reward_time_penalty/tune_ppo.py b/005_rgb_stack_ram_based_reward_time_penalty/tune_ppo.py new file mode 100644 index 0000000..f2d3184 --- /dev/null +++ b/005_rgb_stack_ram_based_reward_time_penalty/tune_ppo.py @@ -0,0 +1,68 @@ +import os + +import retro +import optuna +from stable_baselines3 import PPO +from stable_baselines3.common.monitor import Monitor +from stable_baselines3.common.evaluation import evaluate_policy + +from street_fighter_custom_wrapper import StreetFighterCustomWrapper + +LOG_DIR = 'logs/' +OPT_DIR = 'optuna/' +os.makedirs(LOG_DIR, exist_ok=True) +os.makedirs(OPT_DIR, exist_ok=True) + +def optimize_ppo(trial): + return { + 'n_steps':trial.suggest_int('n_steps', 512, 2048, log=True), + 'gamma':trial.suggest_float('gamma', 0.9, 0.9999), + 'learning_rate':trial.suggest_float('learning_rate', 5e-5, 5e-4, log=True), + 'gae_lambda':trial.suggest_float('gae_lambda', 0.8, 0.9999) + } + +def make_env(game, state): + def _init(): + env = retro.make( + game=game, + state=state, + use_restricted_actions=retro.Actions.FILTERED, + obs_type=retro.Observations.IMAGE + ) + env = StreetFighterCustomWrapper(env) + return env + return _init + +def optimize_agent(trial): + game = "StreetFighterIISpecialChampionEdition-Genesis" + state = "Champion.Level1.ChunLiVsGuile"#"ChampionX.Level1.ChunLiVsKen" + + try: + model_params = optimize_ppo(trial) + + # Create environment + env = make_env(game, state)() + env = Monitor(env, LOG_DIR) + + # Create algo + model = PPO('CnnPolicy', env, verbose=1, **model_params) + model.learn(total_timesteps=500000) + + # Evaluate model + mean_reward, _ = evaluate_policy(model, env, n_eval_episodes=30, deterministic=False) + env.close() + + SAVE_PATH = os.path.join(OPT_DIR, 'trial_{}_best_model'.format(trial.number)) + model.save(SAVE_PATH) + + return mean_reward + + except Exception as e: + return -1 + +# Creating the experiment +study = optuna.create_study(direction='maximize') +study.optimize(optimize_agent, n_trials=10, n_jobs=1) + +print(study.best_params) +print(study.best_trial)