:W&Bá¾&»-ùM ctufruibª; 9 /0.28.1+bc2cf24b1bd9fa3a1b6cbeb920087b832afc526f0.70.0[ò!~‚" vi6o3ecu7dvx ctufruibŠÚ ctufruibescheuller-uc-san-diego spatop-sweep"œ _wandb‚{}  batch_size‚2048  dropout‚0.16234489156944748  hidden_dim‚64 & learning_rate‚0.000139242590613041  num_embedding_layers‚6Bdpfx_blocks_ctufruibbox8or7c7j$dp-spatop-sweepagent-blocks-v2-fb276Š ¥èòÒØš÷š3 (H¨¸hp"3.12.10*0.28.1j linux-x86_64K8À¨; ctufruib*  _wandb‚ {"runtime":0}  _runtime‚0 ¡4U`ÒÚ -Linux-6.8.0-124-generic-x86_64-with-glibc2.36CPython 3.12.10 ¤èòÒ¨ßö¼*-of*>/data/spatop/options_files/sweep_fixed_dp/config_ctufruib.json*-l* /scratch/logs*-n*dpfx_blocks_ctufruib2-m spanet.trainRdanielprimosch@gmail.comZ /scratch/logsb$dp-spatop-sweepagent-blocks-v2-fb276r/opt/conda/bin/pythonº f74vgcdgz0skp3g50v3vco38nra1fudlæPò2  requirements.txtÕ>åêW ctufruibZF (H¨¸ (H¨¸ (hpð"3.12.10*0.28.1j linux-x86_644HY ctufruibZH (H¨¸ (H¨¸ (8hpð"3.12.10*0.28.1j linux-x86_64ù#Lü* ctufruibb trainer/global_step2EpgÇ\  ctufruibZK (H¨¸ (H¨¸(8hpð"3.12.10*0.28.1j linux-x86_64ì„/  ctufruibb*"trainer/global_step2ò—¤ËK  ctufruibj: ¥èòÒ¨¬µ*LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0] JDeviceStatsMonitor.on_train_batch_start/allocation.all.current"trainer/global_step2:JPçd¾âW3bS 0DeviceStatsMonitor.on_train_batch_start/num_ooms"trainer/global_step2:JPþÁ%i4be BDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed"trainer/global_step2:JPf`i5be BDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current"trainer/global_step2:JP·yaln6bj GDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed"trainer/global_step2:JPòˆ)g7bc @DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated"trainer/global_step2:JPX:/‹n8bj GDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated"trainer/global_step2:JPt«—ƒa9b] :DeviceStatsMonitor.on_train_batch_start/active.all.current"trainer/global_step2:JP)_9ƒh:bd ADeviceStatsMonitor.on_train_batch_start/active.small_pool.current"trainer/global_step2:JP ¢ùdp;bl IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated"trainer/global_step2:JPÇäˆ3sba >DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak"trainer/global_step2:JPÓ³O>l?bh EDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current"trainer/global_step2:JP%_åf@bb ?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed"trainer/global_step2:JP;_20iAbe BDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current"trainer/global_step2:JP¶ F‚sBbo LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak"trainer/global_step2:JPŠB£mCbi FDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak"trainer/global_step2:JPü7ÕdkDbg DDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak"trainer/global_step2:JP•È>ugEbc @DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak"trainer/global_step2:JPá}DØnFbj GDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated"trainer/global_step2:JPŒÇÝlGbh EDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak"trainer/global_step2:JPº@MfHbb ?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed"trainer/global_step2:JPÙq+koIbk HDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed"trainer/global_step2:JPTAópJbl IDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current"trainer/global_step2:JPK·+ßdKb` =DeviceStatsMonitor.on_train_batch_start/segment.all.allocated"trainer/global_step2:JP ÿhLbd ADeviceStatsMonitor.on_train_batch_start/active.large_pool.current"trainer/global_step2:JP°ÂexMbt QDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated"trainer/global_step2:JP,š›gNbc @DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed"trainer/global_step2:JP1ü¹ZiObe BDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current"trainer/global_step2:JPïͬÒvPbr ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current"trainer/global_step2:JP!2±mQbi FDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak"trainer/global_step2:JP2”sRbo LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated"trainer/global_step2:JP»'”ÜnSbj GDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed"trainer/global_step2:JP+…–eTba >DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak"trainer/global_step2:JPŽsSjUbf CDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated"trainer/global_step2:JPï¸åÏoVbk HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed"trainer/global_step2:JP³§¨sWbo LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak"trainer/global_step2:JP‰4TÐaXb] :DeviceStatsMonitor.on_train_batch_start/num_oom_rejections"trainer/global_step2:JPrYbn KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated"trainer/global_step2:JPn1nfZbb ?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed"trainer/global_step2:JP¼{eOl[bh EDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed"trainer/global_step2:JPèµq\bm JDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current"trainer/global_step2:JP8[º c]b_ DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed"trainer/global_step2:JPÔ/öfpbb ?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak"trainer/global_step2:JPF ±Ùhqbd ADeviceStatsMonitor.on_train_batch_start/oversize_segments.current"trainer/global_step2:JPðºnRgrbc @DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak"trainer/global_step2:JPQãtÙfsbb ?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak"trainer/global_step2:JP*> _tb[ 8DeviceStatsMonitor.on_train_batch_start/active.all.freed"trainer/global_step2:JPg=¤subo LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated"trainer/global_step2:JPár™öjvbf CDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current"trainer/global_step2:JPO.ñ?jwbf CDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed"trainer/global_step2:JP¶ýB½mxbi FDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed"trainer/global_step2:JP ">^ybZ 7DeviceStatsMonitor.on_train_batch_start/num_device_free"trainer/global_step2:JP á¡–czb_ 2j‘be BDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak"trainer/global_step2:JPV»›çs’bn KDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated"trainer/global_step2:JP‘}1yn“bi FDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak"trainer/global_step2:JP¯¼¦r”bm JDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current"trainer/global_step2:JPø;€o•bj GDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak"trainer/global_step2:JPõÇ{Y^–bY 6DeviceStatsMonitor.on_train_batch_start/max_split_size"trainer/global_step2:JP½Xñ=t—bo LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated"trainer/global_step2:JP¿ªl˜bg DDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated"trainer/global_step2:JPôÕUvf™ba >DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak"trainer/global_step2:JPÏ:Üpšbk HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed"trainer/global_step2:JPŠÐ êl›bg DDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated"trainer/global_step2:JP¢Å*Äqœbl IDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current"trainer/global_step2:JP‹'jí.) _wandbruntime‚33  _runtime‚33;µ™ôמþN @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚115330 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚ 3732946890512 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 _step‚1 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11950 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ]MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 41567936000 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚ 3776652512768 ODDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚369431 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚ 2852144491008 aQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 41684976640 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267415743 NCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚385423 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88480256 WEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚ 3816769769984 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚ 3772951413700 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269120000 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 211150336 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269120000 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12021 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚182 % _timestamp‚1.7844603594360368e+09 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9794 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚254097 K@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚885499 RGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚385423 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9794 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9792 YIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 40100217856 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚ 3776669552128 NCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚385421 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11950 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 J?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚385421 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚186 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 132997632 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 trainer/global_step‚49 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301696512 RGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚500076 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2370606080 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 XHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 39900698930 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚ 2852238600704 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 117040640 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚ 3732963929872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 NCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚500076 XHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 40011737600 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 J?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚490284 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚ 3776652512768 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88480256 GDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚ 3816664250368 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚801 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106882639 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 C8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚875705 \LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 40100217856 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103824258 NCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚490284 XFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚ 2893712427008 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 _runtime‚ 33.826010575 GDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚ 3776652512768 ODDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚369431 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚ 2852144491008 aQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 41684976640 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267415743 NCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚385423 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88480256 WEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚ 3816769769984 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚ 3772951413700 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269120000 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 211150336 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269120000 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12021 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚182 % _timestamp‚1.7844603594360368e+09 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9794 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚254097 K@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚885499 RGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚385423 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9794 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9792 YIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 40100217856 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚ 3776669552128 NCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚385421 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11950 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 J?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚385421 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚186 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 132997632 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 trainer/global_step‚49 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301696512 RGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚500076 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2370606080 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 XHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 39900698930 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚ 2852238600704 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 117040640 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚ 3732963929872 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 NCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚500076 XHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 40011737600 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 J?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚490284 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚ 3776652512768 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88480256 GDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚ 3816664250368 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚801 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106882639 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 C8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚875705 \LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 40100217856 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103824258 NCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚490284 XFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚ 2893712427008 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 _runtime‚ 33.826010575 GDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated"trainer/global_step2:JP!U-e«b` =DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed"trainer/global_step2:JPê¿áh¬bc @DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated"trainer/global_step2:JPú¾c­b^ ;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak"trainer/global_step2:JPÛhhßg®bb ?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current"trainer/global_step2:JPvR³.m¯bh EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed"trainer/global_step2:JPLe;ˆl°bg DDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated"trainer/global_step2:JPèKƒCm±bh EDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak"trainer/global_step2:JPà÷ ój²be BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated"trainer/global_step2:JP­£ÆŽf³ba >DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current"trainer/global_step2:JP]“EÇ]´bX 5DeviceStatsMonitor.on_train_batch_end/active.all.peak"trainer/global_step2:JP~ºô„jµbe BDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak"trainer/global_step2:JPµû•k¶bf CDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed"trainer/global_step2:JP—ŒÂÌh·bc @DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current"trainer/global_step2:JP¾Áåëf¸ba >DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed"trainer/global_step2:JPèoߺm¹bh EDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current"trainer/global_step2:JPwoLsmºbh EDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current"trainer/global_step2:JPžd»b_ DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed"trainer/global_step2:JP– eÄb` =DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed"trainer/global_step2:JP°RFFmÅbh EDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak"trainer/global_step2:JPÔ´û#qÆbl IDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated"trainer/global_step2:JP¤FlaÇb\ 9DeviceStatsMonitor.on_train_batch_end/segment.all.current"trainer/global_step2:JPuÿn^ÈbY 6DeviceStatsMonitor.on_train_batch_end/segment.all.peak"trainer/global_step2:JP>·odÉb_ pØbk HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated"trainer/global_step2:JP.þ­ŒwÙbr ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated"trainer/global_step2:JP÷2g¢\ÚbW 4DeviceStatsMonitor.on_train_batch_end/max_split_size"trainer/global_step2:JPUAMPlÛbg DDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed"trainer/global_step2:JP²ñ°‹uÜbp MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current"trainer/global_step2:JPÄ­QƒbÝb] :DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams"trainer/global_step2:JPðš¾jÞbe BDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current"trainer/global_step2:JP™üúç_ßbZ 7DeviceStatsMonitor.on_train_batch_end/segment.all.freed"trainer/global_step2:JP„âiàbd ADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed"trainer/global_step2:JP‘|»Ajábe BDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak"trainer/global_step2:JP‚Ktu^âbY 6DeviceStatsMonitor.on_train_batch_end/active.all.freed"trainer/global_step2:JPëØÜÚiãbd ADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated"trainer/global_step2:JPygR8gäbb ?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current"trainer/global_step2:JP ¯‹]åbX 5DeviceStatsMonitor.on_train_batch_end/num_device_free"trainer/global_step2:JPÓjgæbb ?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak"trainer/global_step2:JP@ç«`rçbm JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated"trainer/global_step2:JPýl–hnèbi FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed"trainer/global_step2:JP.¹þ^ébY 6DeviceStatsMonitor.on_train_batch_end/num_device_alloc"trainer/global_step2:JP¨kfêba >DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed"trainer/global_step2:JPJ0nhëbc @DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current"trainer/global_step2:JP±¶#œfìba >DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak"trainer/global_step2:JPÅÀqíbl IDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated"trainer/global_step2:JP]©2·qîbl IDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated"trainer/global_step2:JPnö}dïb_ DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak"trainer/global_step2:JPÆÖFlúbg DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak"trainer/global_step2:JPu(cûb^ ;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated"trainer/global_step2:JPGx `üb[ 8DeviceStatsMonitor.on_train_batch_end/active.all.current"trainer/global_step2:JPz×2nýbi FDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current"trainer/global_step2:JPHQ…’eþb` =DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak"trainer/global_step2:JP$²mÿbh EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed"trainer/global_step2:JPÌΤyp€bk HDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current"trainer/global_step2:JPð>Îäkbf CDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak"trainer/global_step2:JPÇ@Kh‚bc @DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current"trainer/global_step2:JPô0_bƒb] :DeviceStatsMonitor.on_train_batch_end/active.all.allocated"trainer/global_step2:JP¯$F!g„bb ?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current"trainer/global_step2:JPKý{%n…bi FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed"trainer/global_step2:JPõ/= l†bg DDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak"trainer/global_step2:JPy×;k‡bf CDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current"trainer/global_step2:JP‚yUmˆbh EDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed"trainer/global_step2:JPdîs‰bn KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed"trainer/global_step2:JP÷=F~mŠbh EDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated"trainer/global_step2:JP|Åe‹b` =DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak"trainer/global_step2:JPé[6%rŒbm JDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak"trainer/global_step2:JPókƒ·nbi FDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed"trainer/global_step2:JPA^ÉzoŽbj GDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current"trainer/global_step2:JP·fµibd ADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated"trainer/global_step2:JP:K¾Yrbm JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated"trainer/global_step2:JP8 LYm‘bh EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak"trainer/global_step2:JPûô«Åj’be BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated"trainer/global_step2:JP…ߊœp“bk HDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current"trainer/global_step2:JPÝ^zße”b` =DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak"trainer/global_step2:JPd™“!i•bd ADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current"trainer/global_step2:JPÙþ’^h–bc @DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak"trainer/global_step2:JPøòççq—bl IDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated"trainer/global_step2:JP6bYùf˜ba >DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed"trainer/global_step2:JP3sìo™bj GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated"trainer/global_step2:JP¦ÿ¾gbšb] :DeviceStatsMonitor.on_train_batch_end/allocation.all.freed"trainer/global_step2:JPÀ Γ.›) _wandbruntime‚34  _runtime‚34v‘ƒ mœˆM PEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚393052 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106882647 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9808 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 209046016 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2370606080 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 E:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚902962 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 VFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 40797453312 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9806 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚259153 [KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 42378693632 PEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚509910 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚801 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚ 3807237415904 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 LADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚393052 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚ 3851819218432 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269121024 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88487424 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105526784 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11952 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚ 3851819218432 E:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚893154 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 _step‚2 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚902962 H=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚500104 R@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚ 3892705159168 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301697536 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105526784 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12023 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 SCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 40797453312 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚183 I>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚376542 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88487424 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚ 3848025433804 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86785010 ZJDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 40885940736 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 MBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚376725 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚393050 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267415751 trainer/global_step‚49 _runtime‚ 34.227856161 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 N6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301697536 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚259157 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚117568 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚393050 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267415751 trainer/global_step‚49 _runtime‚ 34.227856161 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 N6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301697536 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚259157 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚117568 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106882647 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9808 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 209046016 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2370606080 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 E:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚902962 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 VFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 40797453312 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9806 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚259153 [KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 42378693632 PEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚509910 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚801 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚ 3807237415904 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 LADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚393052 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚ 3851819218432 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269121024 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88487424 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105526784 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11952 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚ 3851819218432 E:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚893154 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 _step‚2 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚902962 H=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚500104 R@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚ 3892705159168 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301697536 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105526784 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12023 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 SCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 40797453312 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚183 I>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚376542 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88487424 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚ 3848025433804ÇVJAžb< loss/FRt1/assignment_loss"trainer/global_step2:JP>ÕCŸb> loss/SRqqt1/assignment_loss"trainer/global_step2:JPÖ_âB b= loss/SRqqt2/detection_loss"trainer/global_step2:JPÎâ9C¡b> loss/SRqqt2/assignment_loss"trainer/global_step2:JP/*P†@¢b; loss/FRt2/detection_loss"trainer/global_step2:JP{ 1A£b< loss/FBt1/assignment_loss"trainer/global_step2:JP#›G}A¤b< loss/FBt2/assignment_loss"trainer/global_step2:JPO ¤@¥b; loss/FRt1/detection_loss"trainer/global_step2:JPå† @¦b; loss/FBt2/detection_loss"trainer/global_step2:JPà™ÏW-§b( epoch"trainer/global_step2:JPõ¤A¨b< loss/FRt2/assignment_loss"trainer/global_step2:JPòøåB©b= loss/SRqqt1/detection_loss"trainer/global_step2:JPƒ51@ªb; loss/FBt1/detection_loss"trainer/global_step2:JPžúî7«b2 loss/total_loss"trainer/global_step2:JPt=›.¬) _wandbruntime‚34  _runtime‚34!Tgï÷­ñ %loss/total_loss‚65.77050981766368 trainer/global_step‚49 /loss/FRt1/assignment_loss‚4.909318280669878 1loss/SRqqt1/assignment_loss‚4.220666059807165 0loss/SRqqt2/detection_loss‚2.143298288909834 % _timestamp‚1.7844603598403842e+09 _step‚3 0loss/SRqqt2/assignment_loss‚4.22096870508753 /loss/FRt2/detection_loss‚1.7580530491233999 /loss/FBt1/assignment_loss‚2.179704212280641 /loss/FBt2/assignment_loss‚2.133087707417352 /loss/FRt1/detection_loss‚1.6719603834929813 /loss/FBt2/detection_loss‚17.974629189109518 epoch‚0 _runtime‚ 34.229268454 /loss/FRt2/assignment_loss‚4.950357264030961 1loss/SRqqt1/detection_loss‚2.8211927738970024 /loss/FBt1/detection_loss‚16.7872739038374060à“Šû®õ % _timestamp‚1.7844603598403842e+09 _step‚3 0loss/SRqqt2/assignment_loss‚4.22096870508753 /loss/FRt2/detection_loss‚1.7580530491233999 /loss/FBt1/assignment_loss‚2.179704212280641 /loss/FBt2/assignment_loss‚2.133087707417352 /loss/FRt1/detection_loss‚1.6719603834929813 /loss/FBt2/detection_loss‚17.974629189109518 epoch‚0 _runtime‚ 34.229268454 /loss/FRt2/assignment_loss‚4.950357264030961 1loss/SRqqt1/detection_loss‚2.8211927738970024 /loss/FBt1/detection_loss‚16.787273903837406 %loss/total_loss‚65.77050981766368 trainer/global_step‚49 /loss/FRt1/assignment_loss‚4.909318280669878 1loss/SRqqt1/assignment_loss‚4.220666059807165 0loss/SRqqt2/detection_loss‚2.143298288909834™G3¯‚(:Š ÒèòÒ’ç¤ disk.rbd0.out‚0 disk.nvme0c0n1.in‚0* proc.memory.percent‚17.031359672546387 proc.cpu.threads‚17 disk.nvme1c1n1.out‚55.75 disk.nvme1n1.in‚4 disk.md0.out‚ 1.28515625 disk.dm-1.in‚0 disk.dm-1.out‚ 1.3984375 disk.rbd0.in‚0 disk.nvme0n1.in‚0 disk.nvme1c1n1.in‚4 disk.md0.in‚0 disk.dm-0.out‚259.2490234375# memory_percent‚98.0168104171753 network.sent‚52393 network.recv‚50978) disk./.usagePercent‚75.62746123447613% disk.nvme0c0n1.out‚207.6552734375# disk.nvme0n1.out‚207.6552734375( proc.memory.availableMB‚ 324.92578125# proc.memory.rssMB‚ 2790.41796875 cpu‚25.541441287544593$ disk./.usageGB‚1161.070598602295 disk.nvme1n1.out‚55.75 disk.dm-0.in‚4Ä¿¯rž°‚(:“ ÒèòÒ¾ÕÄ¥ gpu.0.gpu‚61.0 gpu.0.memory‚15, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚47.0 gpu.0.powerWatts‚216.448' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚72.14933333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ò8)±‚(:ž áèòÒ¤Ö¤# disk./.usageGB‚1161.81787109375 disk.md0.out‚ 1.51953125! disk.dm-0.out‚883.10400390625 disk.dm-1.in‚0# memory_percent‚98.0205774307251! proc.memory.rssMB‚ 2790.421875* proc.memory.percent‚17.031383514404297 network.recv‚55328 disk.md0.in‚0 disk.dm-0.in‚4 disk.dm-1.out‚ 2.3095703125 disk.rbd0.in‚0% disk.nvme0c0n1.out‚796.4482421875 network.sent‚101008% disk.nvme1c1n1.out‚92.25537109375 disk.nvme1n1.in‚4# disk.nvme1n1.out‚92.25537109375 disk.rbd0.out‚0# disk.nvme0n1.out‚796.4482421875 cpu‚25.624968889657488) disk./.usagePercent‚75.67613555406285 disk.nvme1c1n1.in‚4 disk.nvme0c0n1.in‚0 disk.nvme0n1.in‚0( proc.memory.availableMB‚ 324.30859375 proc.cpu.threads‚17Fg9¸ž²‚(:“ áèòÒŒ²‡¥ gpu.0.gpu‚60.0 gpu.0.memory‚15, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚46.0 gpu.0.powerWatts‚111.932' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚37.31066666666667 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0«i–T.³) _wandbruntime‚66  _runtime‚668yz£Á´» & lr-AdamW/pg2‚3.446254117672765e-05 trainer/global_step‚99 % _timestamp‚1.7844603918600266e+09 _runtime‚ 66.249115278 _step‚4 & lr-AdamW/pg1‚3.446254117672765e-05Ñ2£®Åµ¿ _step‚4 & lr-AdamW/pg1‚3.446254117672765e-05 & lr-AdamW/pg2‚3.446254117672765e-05 trainer/global_step‚99 % _timestamp‚1.7844603918600266e+09 _runtime‚ 66.249115278êË.¶) _wandbruntime‚66  _runtime‚66’*d~ˆ'·‚O GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 NCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚766871 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9792 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚506901 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12023 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚ 7534152867328 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚ 7534152867328 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚172 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚226147 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚ 7447490199472 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301697536 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚ 7534135827968 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚225979 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚1759801 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88480256 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269121024 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9794 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9792 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 _step‚5 J?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚766871 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 \LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 79450158080 XFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚ 5741240506880 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 WEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚ 7526716214108 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 WEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚ 7613603025408 XHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 79361677824 RGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚766873 aQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 82320365056 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 110749184 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103824258 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86784898 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚ 7447473160112 YIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 79450158080 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269121024 ODDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚733048 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚ 5659030891008 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 HDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9794 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚468 ]MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 82209615872 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 \LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 79226014636 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105519616 \JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚ 5741445365760 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 132997632 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11952 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 _runtime‚ 66.249173278 P>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚ 7613497505792 SADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚ 7613497505792 K@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚732876 SADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚ 7526612389850 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚ 7534135827968 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267415751 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2370606080 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 J?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚983136 UEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 79361677824 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301697536 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11952 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚803 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105519616 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 trainer/global_step‚99 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106882647 HDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9794 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚468 ]MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 82209615872 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 \LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 79226014636 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105519616 \JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚ 5741445365760 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 132997632 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11952 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 _runtime‚ 66.249173278 P>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚ 7613497505792 SADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚ 7613497505792 K@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚732876 SADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚ 7526612389850 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚ 7534135827968 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267415751 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2370606080 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 J?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚983136 UEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 79361677824 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301697536 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11952 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 O§f¼€?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚803 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105519616 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 trainer/global_step‚99 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106882647 HDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12023 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚ 7534152867328 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚ 7534152867328 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚172 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚226147 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚ 7447490199472 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301697536 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚ 7534135827968 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚225979 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚1759801 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88480256 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269121024 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9794 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9792 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 _step‚5 J?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚766871 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 \LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 79450158080 XFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚ 5741240506880 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 WEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚ 7526716214108 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 WEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚ 7613603025408 XHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 79361677824 RGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚766873 aQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 82320365056 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 110749184 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103824258 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86784898 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚ 7447473160112 YIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 79450158080 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269121024 ODDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚733048 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚ 5659030891008 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 HDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 trainer/global_step‚99 LADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚774502 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚ 7609285494272 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88487424 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚1002762 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 213240320 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚1777264 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11952 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚ 7601790234212 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚511953 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9806 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 % _timestamp‚1.7844603922732983e+09 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 _step‚6 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9806 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚ 7521763685504 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚179 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚511957 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 R@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚ 7689538414592 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚ 7521780724864 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 80147393536 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚175 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86785010 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚1777264 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚1767456 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301697536 ZJDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 80235880960 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 119130624 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 VFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 79922724338 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105526784 UCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚ 7689538414592 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2370606080 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚ 7609285494272 Q?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚ 7689432887808 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚ 5715168619008 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103824370 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106882647 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 I>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚740150 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9808 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚ 7609302533632 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 Q?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚ 7601686409842 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚1002762 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88487424 VDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚ 5798186698752J[Rá˜&»’M SCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 80147393536 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 H=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚992956 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚ 7609302533632 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚1777264 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11952 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚ 7601790234212 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚511953 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9806 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 % _timestamp‚1.7844603922732983e+09 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 _step‚6 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9806 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚ 7521763685504 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚179 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚511957 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 R@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚ 7689538414592 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚ 7521780724864 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 80147393536 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚175 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86785010 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚1777264 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚1767456 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301697536 ZJDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 80235880960 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 119130624 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 VFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 79922724338 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105526784 UCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚ 7689538414592 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2370606080 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚ 7609285494272 Q?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚ 7689432887808 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚ 5715168619008 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103824370 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106882647 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 I>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚740150 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9808 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚ 7609302533632 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 Q?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚ 7601686409842 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚1002762 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88487424 VDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚ 5798186698752 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269121024 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚803 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 trainer/global_step‚99 LADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚774502 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚ 7609285494272 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88487424 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚1002762 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 213240320 Fi´ÇÅÁ & lr-AdamW/pg1‚5.186786500335778e-05 & lr-AdamW/pg2‚5.186786500335778e-05 trainer/global_step‚149 % _timestamp‚1.7844604256425745e+09 _runtime‚ 100.032135509 _step‚8¶x”0Æ+ _wandbruntime‚100  _runtime‚100<'д Ç¢O GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 % _timestamp‚1.7844604256452043e+09 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301697536 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚1148323 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚1148321 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 118711618048 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 118377760546 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 118800098304 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12023 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚1485780 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 HDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 118711618048 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9794 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚759697 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 118800098304 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚11291636182528 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚336180 XFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚ 8588700241920 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚1475988 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚183 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚11410330761216 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11952 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚759701 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚11291619143168 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88480256 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11952 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚11161999429712 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105519616 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚11280481014516 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚11280377190258 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88480256 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚336359 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚11410330761216 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11952 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚759701 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚11291619143168 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88480256 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11952 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚11161999429712 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105519616 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚11280481014516 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚11280377190258 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88480256 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚336359 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 118711618048 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9794 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚759697 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 118800098304 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚11291636182528 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚336180 XFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚ 8588700241920 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚1475988 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚183 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 È×L¬0É+ _wandbruntime‚100  _runtime‚100IQdd²&ʬM YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚11236289955104 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269121024 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚11486371670016 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚11486266143232 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚764753 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9808 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12023 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚1485808 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106882647 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚2641758 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚1155950 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚1495614 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚1155950 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚1155952 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 129616384 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 123590259712 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105526784 ODeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88487424 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 119248040156 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 ZHDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚ 8645869004800 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚11355555034620 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269121024 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 137190912 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301697536 _runtime‚ 100.489989504 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 223726080 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 119161255146 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚2651566 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11952 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚ 8522149128704 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 119497333760 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚172 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚11355451210250 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚1103095 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚11486371670016 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚1495614 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚11366785848832 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301697536 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12023 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 123719876096 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267415751 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚1155952 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88487424 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚ 8522055019008 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚11366768809472 trainer/global_step‚149 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9806 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚1485808 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103824370 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 119585821184 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚2651566 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _step‚10 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 119585821184 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 119497333760 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚338510 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚338342 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2370606080 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 $ _timestamp‚1.784460426101245e+09»þȼË°M UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 ZHDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚ 8645869004800 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚11355555034620 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269121024 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 137190912 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301697536 _runtime‚ 100.489989504 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 223726080 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 119161255146 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚2651566 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11952 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚ 8522149128704 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 119497333760 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚172 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚11355451210250 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚1103095 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚11486371670016 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚1495614 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚11366785848832 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301697536 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12023 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 `ODevi 0úceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 123719876096 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267415751 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚1155952 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88487424 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚ 8522055019008 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚11366768809472 trainer/global_step‚149 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9806 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚1485808 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103824370 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 119585821184 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚2651566 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _step‚10 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 119585821184 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 119497333760 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚338510 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚338342 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2370606080 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 $ _timestamp‚1.784460426101245e+09 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚11236289955104 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269121024 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚11486371670016 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚11486266143232 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚764753 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9808 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12023 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚1485808 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106882647 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚2641758 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚1155950 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚1495614 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚1155950 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚1155952 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 129616384 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 123590259712 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105526784 ODeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88487424 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 119248040156 x÷ »0Ì+ _wandbruntime‚100  _runtime‚100š )ÃøÍò 0loss/FRt2/assignment_loss‚5.3294041450010905 .loss/FRt2/detection_loss‚1.532180130900068 0loss/SRqqt1/detection_loss‚5.275698527577786 1loss/SRqqt2/detection_loss‚1.2163854136285701 -loss/FBt1/detection_loss‚33.2739363479905 _runtime‚ 100.491180706 2loss/SRqqt2/assignment_loss‚4.1721862879950855 trainer/global_step‚149 $ _timestamp‚1.784460426102742e+09 _step‚11 /loss/FRt1/assignment_loss‚4.551323846235114 .loss/FBt2/detection_loss‚8.093549431244437 epoch‚0 0loss/SRqqt1/assignment_loss‚4.22864793339592 /loss/FBt1/assignment_loss‚1.351066752177913 0loss/FBt2/assignment_loss‚1.4028329905562513 /loss/FRt1/detection_loss‚0.4382835241173733 %loss/total_loss‚70.86549533082011¼ËëüÎö _step‚11 /loss/FRt1/assignment_loss‚4.551323846235114 .loss/FBt2/detection_loss‚8.093549431244437 epoch‚0 0loss/SRqqt1/assignment_loss‚4.22864793339592 /loss/FBt1/assignment_loss‚1.351066752177913 0loss/FBt2/assignment_loss‚1.4028329905562513 /loss/FRt1/detection_loss‚0.4382835241173733 %loss/total_loss‚70.86549533082011 0loss/FRt2/assignment_loss‚5.3294041450010905 .loss/FRt2/detection_loss‚1.532180130900068 0loss/SRqqt1/detection_loss‚5.275698527577786 1loss/SRqqt2/detection_loss‚1.2163854136285701 -loss/FBt1/detection_loss‚33.2739363479905 _runtime‚ 100.491180706 2loss/SRqqt2/assignment_loss‚4.1721862879950855 trainer/global_step‚149 $ _timestamp‚1.784460426102742e+09 CK $2Ï‚(:§ ŽéòÒžŠ÷¤$ memory_percent‚98.00734519958496' proc.memory.availableMB‚ 326.4765625 cpu‚25.613255967360757% disk.nvme0n1.out‚1181.28466796875 disk.nvme1c1n1.in‚4 disk.dm-0.in‚4" proc.memory.rssMB‚ 2790.4296875% disk./.usageGB‚1164.5817108154297 disk.rbd0.in‚0 disk.nvme0n1.in‚0& disk.nvme1c1n1.out‚2007.4990234375 disk.md0.in‚0 disk.md0.out‚ 2.51953125* proc.memory.percent‚17.031431198120117 proc.cpu.threads‚17 network.recv‚66552 disk.rbd0.out‚0 disk.nvme0c0n1.in‚0' disk.nvme0c0n1.out‚1181.28466796875 disk.nvme1n1.in‚4$ disk.nvme1n1.out‚2007.4990234375 disk.dm-0.out‚3180.736328125 disk.dm-1.in‚0 network.sent‚169806) disk./.usagePercent‚75.85616093896301 disk.dm-1.out‚ 2.59912109375ztÛèžÐ‚(:“ ŽéòÒò窥 gpu.0.gpu‚98.0 gpu.0.memory‚23, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚46.0 gpu.0.powerWatts‚112.153' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚37.38433333333334 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0–‰‘D1Ñ‚(:¦ éòÒàÃì¤ disk.nvme0n1.in‚0$ memory_percent‚98.00879955291748( proc.memory.availableMB‚ 326.23828125" proc.memory.rssMB‚ 2790.4296875% disk.nvme1n1.out‚2008.27880859375 disk.dm-0.in‚4! disk.dm-0.out‚3184.0693359375 disk.rbd0.in‚0 disk.rbd0.out‚0 disk.nvme0c0n1.in‚0 proc.cpu.threads‚17 network.sent‚204782 network.recv‚70633% disk./.usageGB‚1164.5822715759277 disk.nvme1c1n1.in‚4 disk.dm-1.in‚0 disk.dm-1.out‚ 2.81396484375* proc.memory.percent‚17.031431198120117) disk./.usagePercent‚75.85619746464275 disk.nvme1n1.in‚4 disk.md0.in‚0 disk.md0.out‚2.765625& disk.nvme0c0n1.out‚1184.5888671875$ disk.nvme0n1.out‚1184.5888671875 cpu‚25.61180067510069' disk.nvme1c1n1.out‚2008.27880859375XèdóÒ‚(:’ éòÒ¦™ ¥ gpu.0.gpu‚36.0 gpu.0.memory‚9, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚47.0 gpu.0.powerWatts‚112.391' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚37.46366666666667 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0§öj0Ó+ _wandbruntime‚133  _runtime‚133'M÷ºÂÔ¼ _step‚12 % lr-AdamW/pg1‚6.92731888299879e-05 % lr-AdamW/pg2‚6.92731888299879e-05 trainer/global_step‚199 % _timestamp‚1.7844604588328078e+09 _runtime‚ 133.221195988òDäÆÕÀ % _timestamp‚1.7844604588328078e+09 _runtime‚ 133.221195988 _step‚12 % lr-AdamW/pg1‚6.92731888299879e-05 % lr-AdamW/pg2‚6.92731888299879e-05 trainer/global_step‚199 Ö† 0Ö+ _wandbruntime‚133  _runtime‚133%Õù¬'צO A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12023 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚15049102458368 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚11436372611072 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 163474810368 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚3498611 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚1458094 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚3508405 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚14876525699312 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11952 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚445593 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11952 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 137190912 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103824258 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 157616291354 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚1978632 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 _step‚13 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚15207269536256 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9794 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105519616 trainer/global_step‚199 # _timestamp‚1.78446045883386e+09 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 158061558272 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚1529773 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269121024 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9792 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 217441792 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 157703076252 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚1529771 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚1457921 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9794 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚11272897800704 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚1968840 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚15207164016640 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚15049119497728 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 _runtime‚ 133.22221327 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚11272803691008 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267415751 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 HDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 158061558272 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚1529771 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚15049119497728 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚15034245814924 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301697536 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88480256 HDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11952 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚445593 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11952 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 137190912 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103824258 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 157616291354 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚1978632 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 _step‚13 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚15207269536256 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9794 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105519616 trainer/global_step‚199 # _timestamp‚1.78446045883386e+09 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 158061558272 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚1529773 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269121024 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9792 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 217441792 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 157703076252 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚1529771 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚1457921 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9794 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚11272897800704 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚1968840 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚15207164016640 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚15049119497728 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 _runtime‚ 133.22221327 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚11272803691008 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267415751 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 HDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 158061558272 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚1529771 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚15049119497728 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚15034245814924 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301697536 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88480256 HDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚1465211 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 trainer/global_step‚199 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚3516060 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚1988466 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301697536 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚1978660 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 158935761408 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106882647 E?DeviceStatsMoni˜‡¾×Xtor.on_train_batch_end/active.large_pool.current‚2 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚15283204925440 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11952 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 164287494144 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚14950816224704 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301697536 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚15124269164032 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚1537402 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚1988466 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚1978660 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚15283204925440 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 158847273984 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9806 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2370606080 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103824370 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚3516060 _runtime‚ 133.640128078 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚11329035528704 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86785010 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 158847273984 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12023 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚182 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105526784 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚1537402 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 158399785954 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 158486570964 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚3525868 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚11493097199616 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚1537400 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚3525868 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚447836 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9806 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚15124252124672 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚1017557 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚14950833264064 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚1465393 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 225823232 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267415751 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269121024 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚11493323022848 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚15124269164032 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚1537400 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9808 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚15283099398656 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚11328941419008 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 131713536 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88487424 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88487424 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 _step‚14N7‰^¼&Û¶M QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚1017553 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚15109319835028 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 $ _timestamp‚1.784460459251398e+09 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚178 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚447658 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269121024 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 158935761408 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚1465211 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 trainer/global_step‚199 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚3516060 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚1988466 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301697536 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚1978660 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 158935761408 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106882647 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚15283204925440 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11952 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 164287494144 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚14950816224704 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301697536 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚15124269164032 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚1537402 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚1988466 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚1978660 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚15283204925440 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 158847273984 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9806 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2370606080 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103824370 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚3516060 _runtime‚ 133.640128078 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚11329035528704 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86785010 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 158847273984 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12023 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚182 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105526784 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚1537402 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 158399785954 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 158486570964 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚3525868 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚11493097199616 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚1537400 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚3525868 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚447836 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9806 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚15124252124672 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚1017557 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚14950833264064 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚1465393 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 225823232 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267415751 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269121024 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚11493323022848 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚15124269164032 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚1537400 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9808 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚15283099398656 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚11328941419008 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 131713536 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88487424 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88487424 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 _step‚14w—^$0Ü+ _wandbruntime‚133  _runtime‚1338}í²úÝô 0loss/SRqqt1/detection_loss‚4.601957156060216 trainer/global_step‚199 epoch‚0 /loss/FRt1/assignment_loss‚4.491535790274969 2loss/SRqqt2/assignment_loss‚3.3565305846078055 /loss/FRt2/detection_loss‚1.5880090818326507 .loss/FBt2/detection_loss‚6.192589078294978 %loss/total_loss‚55.31936912805071 _runtime‚ 133.641115481 _step‚15 0loss/SRqqt1/assignment_loss‚3.49300195707943 0loss/FBt2/assignment_loss‚0.3136129188351333 0loss/FRt1/detection_loss‚0.35764131524780784 % _timestamp‚1.7844604592527273e+09 0loss/FBt1/assignment_loss‚0.5682896198704839 1loss/SRqqt2/detection_loss‚1.2107177525825266 -loss/FBt1/detection_loss‚23.8215485388147 .loss/FRt2/assignment_loss‚5.32393533455001>ýûùþÞø -loss/FBt1/detection_loss‚23.8215485388147 .loss/FRt2/assignment_loss‚5.32393533455001 0loss/SRqqt1/detection_loss‚4.601957156060216 trainer/global_step‚199 epoch‚0 /loss/FRt1/assignment_loss‚4.491535790274969 2loss/SRqqt2/assignment_loss‚3.3565305846078055 /loss/FRt2/detection_loss‚1.5880090818326507 .loss/FBt2/detection_loss‚6.192589078294978 %loss/total_loss‚55.31936912805071 _runtime‚ 133.641115481 _step‚15 0loss/SRqqt1/assignment_loss‚3.49300195707943 0loss/FBt2/assignment_loss‚0.3136129188351333 0loss/FRt1/detection_loss‚0.35764131524780784 % _timestamp‚1.7844604592527273e+09 0loss/FBt1/assignment_loss‚0.5682896198704839 1loss/SRqqt2/detection_loss‚1.2107177525825266a zŸ3ß‚(:¨ ¬éòÒ¦ù¤ disk.nvme0n1.in‚0 disk.nvme1c1n1.in‚4! disk.dm-0.out‚3185.0771484375 disk.dm-1.out‚ 2.86865234375$ memory_percent‚98.01821708679199) disk./.usagePercent‚75.85619373753258 disk.rbd0.out‚0$ disk.nvme0n1.out‚1185.9228515625 disk.nvme1n1.in‚4 disk.md0.in‚0 disk.dm-1.in‚0' proc.memory.availableMB‚ 324.6953125* proc.memory.percent‚17.031431198120117 network.sent‚221075 network.recv‚74627" proc.memory.rssMB‚ 2790.4296875% disk./.usageGB‚1164.5822143554688' disk.nvme1c1n1.out‚2008.87841796875% disk.nvme1n1.out‚2008.87841796875 disk.md0.out‚ 3.16796875 disk.dm-0.in‚4 cpu‚25.613939486446196 proc.cpu.threads‚17 disk.rbd0.in‚0 disk.nvme0c0n1.in‚0& disk.nvme0c0n1.out‚1185.9228515625)lQ?à‚(:’ ¬éòÒóÿ¬¥ gpu.0.gpu‚41.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚49.0 gpu.0.powerWatts‚212.405' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚70.80166666666666 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¹’»!,á‚(:¡ »éòÒ²úþ¤! disk.dm-0.out‚4057.7255859375# memory_percent‚98.0194091796875! proc.memory.availableMB‚324.5$ disk.nvme0n1.out‚1676.9482421875 disk.nvme1c1n1.in‚4' disk.nvme1c1n1.out‚2391.27490234375 disk.md0.in‚0 cpu‚25.620292802905347 disk.rbd0.out‚0& disk.nvme0c0n1.out‚1676.9482421875 disk.nvme0n1.in‚0 disk.nvme1n1.in‚4" proc.memory.rssMB‚ 2790.4296875* proc.memory.percent‚17.031431198120117% disk./.usageGB‚1164.5359725952148 disk.dm-1.in‚0 disk.dm-1.out‚ 2.99755859375% disk.nvme1n1.out‚2391.27490234375 disk.md0.out‚ 3.45703125 disk.dm-0.in‚4 proc.cpu.threads‚17 network.sent‚257174 network.recv‚79157) disk./.usagePercent‚75.85318173556195 disk.rbd0.in‚0 disk.nvme0c0n1.in‚0è×Ðâ‚(:’ »éòÒâð³¥ gpu.0.gpu‚85.0 gpu.0.memory‚14, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚49.0 gpu.0.powerWatts‚217.38' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚72.46000000000001 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0NÂõ1ã‚(:¦ ÊéòÒðÞæ¤ disk.rbd0.in‚0 disk.nvme0c0n1.in‚0 disk.nvme0n1.in‚0 disk.md0.out‚3.796875 cpu‚25.66907744445844& disk.nvme0c0n1.out‚1678.1123046875 disk.nvme1n1.in‚4% disk.nvme1n1.out‚2391.82177734375 disk.dm-1.in‚0$ memory_percent‚98.02029132843018( proc.memory.availableMB‚ 324.35546875$ disk.nvme0n1.out‚1678.1123046875 disk.md0.in‚0! disk.dm-0.out‚4058.6552734375" proc.memory.rssMB‚ 2790.4296875* proc.memory.percent‚17.031431198120117 proc.cpu.threads‚17 network.sent‚262456% disk./.usageGB‚1164.5357933044434 disk.rbd0.out‚0 disk.nvme1c1n1.in‚4' disk.nvme1c1n1.out‚2391.82177734375 disk.dm-0.in‚4 disk.dm-1.out‚ 3.03662109375 network.recv‚82946) disk./.usagePercent‚75.85317005728339úIýä‚(:’ ÊéòÒâ« ¥ gpu.0.gpu‚90.0 gpu.0.memory‚18, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚48.0 gpu.0.powerWatts‚112.91' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚37.63666666666667 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ûº!ñ0å+ _wandbruntime‚165  _runtime‚165tª—Äæ¾ & lr-AdamW/pg1‚8.667851265661804e-05 & lr-AdamW/pg2‚8.667851265661804e-05 trainer/global_step‚249 % _timestamp‚1.7844604913730373e+09 _runtime‚ 165.761424585 _step‚16X4þ¥Èç & lr-AdamW/pg1‚8.667851265661804e-05 & lr-AdamW/pg2‚8.667851265661804e-05 trainer/global_step‚249 % _timestamp‚1.7844604913730373e+09 _runtime‚ 165.761424585 _step‚16áÍç0è+ _wandbruntime‚165  _runtime‚165:fÀ±&é©O VKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚554064 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚1911223 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11952 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 _runtime‚ 165.762566675 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12023 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105519616 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚2461692 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚18591069008272 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚2471484 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚18806585773568 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚1911221 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9794 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚1265297 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 % _timestamp‚1.7844604913741498e+09 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269121024 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚14283826542592 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚18806585773568 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88480256 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 197411498496 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9794 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 221636096 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚1265301 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 127526400 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9792 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚553893 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚18787906791074 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚175 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 137190912 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚18806602812928 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 197411498496 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚14079690091008 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚4382707 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103824258 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚1819365 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301697536 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 HDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2370606080 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 196941607060 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269121024 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9792 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚171 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚18788010615332 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚19003997272064 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301697536 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 196854822162 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12023 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚18806602812928 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚1819190 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚14283604906496 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 trainer/global_step‚249 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105519616 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚19003997272064 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 204042341888 E?DeviceStatsMonitor.on_train_batch_start/oversize_segmeI]Ïv®nts.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚19004102791680 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚2461692 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267415751 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0"࿳'ê­O E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚19004102791680 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚2461692 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267415751 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚554064 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚1911223 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11952 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 _runtime‚ 165.762566675 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12023 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105519616 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚2461692 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚18591069008272 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚2471484 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚18806585773568 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚1911221 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9794 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚1265297 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 % _timestamp‚1.7844604913741498e+09 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269121024 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚14283826542592 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚18806585773568 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88480256 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 197411498496 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9794 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 221636096 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚1265301 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 127526400 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9792 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚553893 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚18787906791074 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚175 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 137190912 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚18806602812928 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 197411498496 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚14079690091008 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚4382707 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103824258 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚1819365 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301697536 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 HDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2370606080 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 196941607060 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269121024 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9792 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚171 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚18788010615332 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚19003997272064 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301697536 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 196854822162 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12023 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚18806602812928 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚1819190 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚14283604906496 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 trainer/global_step‚249 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105519616 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚19003997272064 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 204042341888]3š0ë+ _wandbruntime‚166  _runtime‚166--¯ø¹&ì³M J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚1826408 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚4390362 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267415751 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚19080038180864 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚18881752479232 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 198197214208 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 ODeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 _step‚18 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚14340770671104 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105526784 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚1270357 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 137190912 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚19080038180864 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚2471512 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚4400170 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚14135921928704 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 119130624 _runtime‚ 166.186405584 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 197638316762 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚4400170 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301697536 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269121024 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12023 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚556218 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚556055 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚18665359533664 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 197725101772 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚4390362 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚1918852 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12023 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 198197214208 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9806 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚1826575 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚163 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86785010 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚1918850 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2370606080 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚18665342494304 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269121024 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 trainer/global_step‚249 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9806 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106882647 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚167 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 198285701632 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88487424 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚14135827819008 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚2481318 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚1270353 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301697536 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11952 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 204848742400 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 % _timestamp‚1.7844604917978168e+09 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚18881752479232 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚19079932654080 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚2471512 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103824370 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105526784 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚18881735439872 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0ùø?½&í·M MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚1918852 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105526784 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚18881735439872 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚1826408 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚4390362 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267415751 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚19080038180864 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚18881752479232 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 198197214208 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 ODeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 _step‚18 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚14340770671104 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105526784 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚1270357 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 137190912 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚19080038180864 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚2471512 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚4400170 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚14135921928704 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 119130624 _runtime‚ 166.186405584 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 197638316762 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚4400170 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301697536 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269121024 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12023 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚556218 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚556055 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚18665359533664 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 197725101772 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚4390362 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚1918852 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12023 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 198197214208 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9806 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚1826575 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚163 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86785010 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚1918850 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2370606080 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚18665342494304 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269121024 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 trainer/global_step‚249 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9806 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106882647 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚167 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 198285701632 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88487424 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚14135827819008 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚2481318 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚1270353 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301697536 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11952 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 204848742400 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 % _timestamp‚1.7844604917978168e+09 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚18881752479232 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚19079932654080 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚2471512 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103824370 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 EDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚2292673 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9792 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106882647 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9794 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269121024 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚5257009 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11952 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 % _timestamp‚1.7844605236539655e+09 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105519616 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚2180133 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚1518101 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267415751 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚22800936047104 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚22800830527488 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚2292671 trainer/global_step‚299 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚2954544 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 _step‚21 HDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11952 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301697536 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚22564086128128 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9794 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚2292673 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚2179947 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 129623552 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚22800830527488 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9792 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚17131274868736 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚22541671591482 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88480256 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 236180137868 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105519616 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88480256 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚2292671 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86784898W¸Á³'ú­O OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚2292673 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9792 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106882647 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9794 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269121024 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚5257009 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11952 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 % _timestamp‚1.7844605236539655e+09 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105519616 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚2180133 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚1518101 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267415751 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚22800936047104 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚22800830527488 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚2292671 trainer/global_step‚299 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚2954544 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 _step‚21 HDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11952 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301697536 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚22564086128128 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9794 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚2292673 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚2179947 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 129623552 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚22800830527488 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9792 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚17131274868736 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚22541671591482 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88480256 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 236180137868 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105519616 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88480256 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚2292671 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86784898 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚661850 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚2964336 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 236761438720 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269121024 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚22800936047104 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚17131051135488 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚5247215 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚16886670600704 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12023 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚22305578238512 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 236849918976 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2370606080 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚22541775415740 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚186 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚662032 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 236761438720 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301697536 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚16886576491008 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 244604268032 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12023 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚07ùÔ30û+ _wandbruntime‚198  _runtime‚198÷¹¶ã¸&ü²M REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88487424 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚16942714219008 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚5264664 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚2300300 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚1523153 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚664147 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚17187999752192 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚5274472 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2370606080 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚5274472 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚22876765909504 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 _step‚22 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚171 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9806 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301697536 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚22639218755072 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚2300302 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚2974170 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚2964364 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12023 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚22379885803264 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 O6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 _runtime‚ 198.445322526 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 236876847570 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11952 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12023 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚22876871436288 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301697536 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106882647 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚22639218755072 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚2974170 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚22876871436288 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 237635641856 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 236963632580 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 237635641856 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚22616745611474 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 237547154432 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105526784 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚2187304 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 245285533184 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103824370 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚22639235794432 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9806 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚22379868763904 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88487424 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚5264664 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚17188223478272 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚2964364 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚663976 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 129616384 trainer/global_step‚299 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 $ _timestamp‚1.784460524056767e+09 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105526784 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚2187129 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 223726080 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚22616849435844 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269121024 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682ÜŸ33%ý¶M XGDeviceStatsMonitor.on_trainú²ý‰—&_batch_end/active_bytes.small_pool.allocated‚ 237635641856 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚22616745611474 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 237547154432 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105526784 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚2187304 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 245285533184 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103824370 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚22639235794432 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9806 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚22379868763904 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88487424 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚5264664 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚17188223478272 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚2964364 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚663976 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 129616384 trainer/global_step‚299 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 $ _timestamp‚1.784460524056767e+09 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105526784 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚2187129 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 223726080 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚22616849435844 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269121024 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88487424 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚16942714219008 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚5264664 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚2300300 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚1523153 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚664147 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚17187999752192 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚5274472 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2370606080 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚5274472 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚22876765909504 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 _step‚22 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚171 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9806 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301697536 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚22639218755072 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚2300302 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚2974170 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚2964364 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12023 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚22379885803264 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 O6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 _runtime‚ 198.445322526 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 236876847570 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚11952 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12023 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚22876871436288 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301697536 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106882647 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚22639218755072 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚2974170 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚22876871436288 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 237635641856 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 236963632580 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0‡Ù8$0þ+ _wandbruntime‚198  _runtime‚198¢Ã¾Ìüÿö % _timestamp‚1.7844605240577085e+09 .loss/FRt1/assignment_loss‚4.56933116779274 1loss/FBt1/assignment_loss‚0.45709053992938536 .loss/FRt2/detection_loss‚1.450670086904622 _runtime‚ 198.445826196 _step‚23 1loss/SRqqt1/assignment_loss‚2.550489748470367 0loss/FBt2/assignment_loss‚0.2502221256928069 1loss/SRqqt2/detection_loss‚1.1574746023114928 /loss/FBt1/detection_loss‚24.779567492731776 2loss/SRqqt2/assignment_loss‚2.2365406819503675 0loss/SRqqt1/detection_loss‚3.308725922111671 epoch‚0 /loss/FRt2/assignment_loss‚4.728430807989445 /loss/FRt1/detection_loss‚0.4188746770594239 .loss/FBt2/detection_loss‚6.476570921151031 %loss/total_loss‚52.38398877409513 trainer/global_step‚299”Ì®ü€ú 0loss/SRqqt1/detection_loss‚3.308725922111671 epoch‚0 /loss/FRt2/assignment_loss‚4.728430807989445 /loss/FRt1/detection_loss‚0.4188746770594239 .loss/FBt2/detection_loss‚6.476570921151031 %loss/total_loss‚52.38398877409513 trainer/global_step‚299 % _timestamp‚1.7844605240577085e+09 .loss/FRt1/assignment_loss‚4.56933116779274 1loss/FBt1/assignment_loss‚0.45709053992938536 .loss/FRt2/detection_loss‚1.450670086904622 _runtime‚ 198.445826196 _step‚23 1loss/SRqqt1/assignment_loss‚2.550489748470367 0loss/FBt2/assignment_loss‚0.2502221256928069 1loss/SRqqt2/detection_loss‚1.1574746023114928 /loss/FBt1/detection_loss‚24.779567492731776 2loss/SRqqt2/assignment_loss‚2.2365406819503675'!»G‚(:’ ÷éòÒèþÅ¥ gpu.0.gpu‚30.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚49.0 gpu.0.powerWatts‚222.436' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚74.14533333333334 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ŠW-V‚‚(:Ë ÷éòÒáÆì¥ disk.md0.in‚0 disk.md0.out‚ 6.9140625 disk.dm-1.in‚0 disk.rbd0.out‚0! disk.nvme0c0n1.in‚ 55.36328125' disk.nvme0c0n1.out‚3703.84619140625# disk.nvme1c1n1.out‚ 4062.4140625$ memory_percent‚98.02312850952148 network.recv‚95806 disk.dm-0.in‚ 76.43359375" disk.dm-0.out‚7229.22509765625% disk.nvme0n1.out‚3703.84619140625! disk.nvme1n1.out‚ 4062.4140625 proc.memory.rssMB‚ 2790.4375* proc.memory.percent‚17.031478881835938 cpu‚25.671696549738577$ disk./.usageGB‚1169.659408569336 disk.dm-1.out‚ 522.38671875 disk.nvme0n1.in‚ 55.36328125& proc.memory.availableMB‚ 323.890625 proc.cpu.threads‚17 network.sent‚332379( disk./.usagePercent‚76.1869017143355 disk.rbd0.in‚0 disk.nvme1c1n1.in‚ 21.0703125 disk.nvme1n1.in‚ 21.0703125!B“Pƒ‚(:Å †êòÒê߶¤ network.recv‚100241% disk./.usageGB‚1172.3008918762207# disk.nvme1c1n1.out‚ 4121.8203125 disk.md0.out‚7.46875 proc.cpu.threads‚17( disk./.usagePercent‚76.3589572953083 disk.dm-0.out‚ 7394.58203125 disk.dm-1.out‚ 522.8671875$ disk.nvme0c0n1.out‚ 3811.44140625 disk.nvme1c1n1.in‚ 82.7734375 disk.md0.in‚0# memory_percent‚98.0250597000122 proc.memory.rssMB‚ 2790.4375 disk.dm-1.in‚0 disk.rbd0.in‚0 disk.rbd0.out‚0" disk.nvme0n1.out‚ 3811.44140625! disk.nvme1n1.out‚ 4121.8203125* proc.memory.percent‚17.031478881835938 cpu‚25.664362018053964 network.sent‚365621 disk.dm-0.in‚ 704.66796875" disk.nvme0c0n1.in‚ 621.89453125 disk.nvme0n1.in‚ 621.89453125 disk.nvme1n1.in‚ 82.7734375( proc.memory.availableMB‚ 323.57421875‹@žÃž„‚(:“ †êòÒŠ‹ü¤ gpu.0.gpu‚97.0 gpu.0.memory‚23, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚49.0 gpu.0.powerWatts‚154.014' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚51.33800000000001 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0öp)0…+ _wandbruntime‚230  _runtime‚230ÚÐrņ¿ _step‚24 ' lr-AdamW/pg1‚0.00012148916030987829 ' lr-AdamW/pg2‚0.00012148916030987829 trainer/global_step‚349 $ _timestamp‚1.784460555751887e+09 _runtime‚ 230.140756843†wɇà ' lr-AdamW/pg1‚0.00012148916030987829 ' lr-AdamW/pg2‚0.00012148916030987829 trainer/global_step‚349 $ _timestamp‚1.784460555751887e+09 _runtime‚ 230.140756843 _step‚24ÙÁ¶Ã0ˆ+ _wandbruntime‚230  _runtime‚230@|®É­'‰§O G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 VKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚769394 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 127526400 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105519616 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚26597769302528 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚3457188 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚19978681760768 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 HDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚26321552403968 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚2674123 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301697536 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚2540113 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88480256 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269121024 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301697536 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚26295436391890 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚26020104508112 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 NEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚9792 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚26597769302528 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚26597663782912 HDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11952 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚769216 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 221636096 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12023 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚6131311 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚1770901 # _timestamp‚1.78446055575261e+09 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚2540295 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 275418668676 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 275331883778 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚26321569443328 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚182 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 _step‚25 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105519616 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2370606080 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 276199859200 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚178 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚19693462891008 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 276111378944 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12023 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚26597663782912 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚19693557000704 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 276111378944 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚26321552403968 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚3447396 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 137190912 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106882647 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9794 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚26321569443328 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9794 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11952 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 _runtime‚ 230.140974473 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 trainer/global_step‚349 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚3457188 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚1770897 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚6121517 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚2674123 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚2674121 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚19978460124672 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 285124760064 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267415751 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚2674121 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 284997233664 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103824258 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9792 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88480256 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11952 RGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚769216 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 221636096 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12023 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚6131311 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚1770901 # _timestamp‚1.78446055575261e+09 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚2540295 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 275418668676 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 275331883778 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚26321569443328 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚182 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 _step‚25 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105519616 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2370606080 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 276199859200 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚178 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚19693462891008 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 276111378944 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12023 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚26597663782912 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚19693557000704 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 276111378944 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚26321552403968 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚3447396 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 137190912 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106882647 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 C:DeviceStatsMonitor.on_train_batch_start/active.all.current‚9794 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚26321569443328 G>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9794 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚11952 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 _runtime‚ 230.140974473 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 trainer/global_step‚349 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚3457188 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚1770897 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚6121517 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚2674123 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚2674121 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚19978460124672 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 285124760064 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267415751 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚2674121 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 284997233664 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103824258 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 JADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚9792 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88480256 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚26321552403968 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4vÜ=x0‹+ _wandbruntime‚230  _runtime‚230ñU£É·&Œ±M ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12023 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9806 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚187 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚20035406038016 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 276202163388 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86785010 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 276115378378 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚3457216 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301697536 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚26396702070272 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 276897094656 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12023 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚2547300 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚3457216 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2370606080 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚19749694728704 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚6138966 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚3467022 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267415751 trainer/global_step‚349 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9806 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚771347 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 285805419008 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103824370 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚2681750 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚1775953 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106882647 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚6138966 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚26396719109632 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105526784 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚26396719109632 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚26673704691712 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚26370510411882 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9808 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚2681752 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚26673599164928 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚1775957 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 285941326848 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301697536 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚19749600619008 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 135907840 _step‚26 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚6148774 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105526784 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚20035636055552 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚26094412072864 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 % _timestamp‚1.7844605561556492e+09 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚26673704691712 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚6148774 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚26094395033504 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚1307Ý»&µM WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 276897094656 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12023 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚2547300 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚3457216 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2370606080 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚19749694728704 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚6138966 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚3467022 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267415751 trainer/global_step‚349 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9806 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚771347 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 285805419008 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103824370 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚2681750 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚1775953 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106882647 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚6138966 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚26396719109632 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105526784 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚26396719109632 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚26673704691712 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚26370510411882 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 A8DeviceStatsMonitor.on_train_batch_end/active.all.current‚9808 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚2681752 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚26673599164928 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚1775957 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 285941326848 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301697536 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚19749600619008 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 135907840 _step‚26 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚6148774 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105526784 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚20035636055552 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚26094412072864 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 % _timestamp‚1.7844605561556492e+09 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚26673704691712 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚6148774 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚26094395033504 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12023 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9806 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚187 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚20035406038016 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 276202163388 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86785010 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 276115378378 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚3457216 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301697536 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚26396702070272ÆüÑo0Ž+ _wandbruntime‚230  _runtime‚230âX«Ôþø &loss/total_loss‚47.639005153481015 /loss/FRt1/assignment_loss‚4.618000943431052 /loss/FRt2/assignment_loss‚4.590172508098935 /loss/FRt1/detection_loss‚0.5190989854814279 1loss/SRqqt1/detection_loss‚2.4751375596156233 /loss/FBt1/detection_loss‚23.511438481874322 trainer/global_step‚349 _step‚27 0loss/FBt2/assignment_loss‚0.1924644724959733 /loss/FRt2/detection_loss‚1.4739304420657007 epoch‚0 % _timestamp‚1.7844605561565764e+09 2loss/SRqqt1/assignment_loss‚2.2814693312970395 2loss/SRqqt2/assignment_loss‚2.1926630171523853 0loss/FBt1/assignment_loss‚0.2303196257181788 _runtime‚ 230.5446198 1loss/SRqqt2/detection_loss‚1.1313947163454776 .loss/FBt2/detection_loss‚4.422915069904898u´újü /loss/FRt2/assignment_loss‚4.590172508098935 /loss/FRt1/detection_loss‚0.5190989854814279 1loss/SRqqt1/detection_loss‚2.4751375596156233 /loss/FBt1/detection_loss‚23.511438481874322 trainer/global_step‚349 _step‚27 0loss/FBt2/assignment_loss‚0.1924644724959733 /loss/FRt2/detection_loss‚1.4739304420657007 epoch‚0 % _timestamp‚1.7844605561565764e+09 2loss/SRqqt1/assignment_loss‚2.2814693312970395 2loss/SRqqt2/assignment_loss‚2.1926630171523853 0loss/FBt1/assignment_loss‚0.2303196257181788 _runtime‚ 230.5446198 1loss/SRqqt2/detection_loss‚1.1313947163454776 .loss/FBt2/detection_loss‚4.422915069904898 &loss/total_loss‚47.639005153481015 /loss/FRt1/assignment_loss‚4.618000943431052uþ`‘‚(:Õ •êòÒðÁ̤ disk.nvme1c1n1.in‚111.25 proc.memory.rssMB‚ 2790.4375 cpu‚25.65657219206807$ disk./.usageGB‚1174.755069732666 disk.nvme1n1.in‚111.25 disk.md0.in‚0! disk.nvme0n1.in‚ 1686.66015625 network.sent‚372203 network.recv‚104281# disk.dm-0.out‚14071.42041015625 disk.rbd0.in‚0' disk.nvme0c0n1.out‚6266.06787109375' disk.nvme1c1n1.out‚8345.15478515625$ memory_percent‚98.02582263946533( proc.memory.availableMB‚ 323.44921875 disk.rbd0.out‚0( disk./.usagePercent‚76.5188125538294% disk.nvme1n1.out‚8345.15478515625 disk.md0.out‚7.84375 disk.dm-0.in‚ 1797.91015625 disk.dm-1.in‚0* proc.memory.percent‚17.031478881835938 proc.cpu.threads‚17! disk.dm-1.out‚523.19482421875# disk.nvme0c0n1.in‚ 1686.66015625% disk.nvme0n1.out‚6266.06787109375E×ë'ž’‚(:“ •êòÒ„⥥ gpu.0.gpu‚91.0 gpu.0.memory‚16, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚50.0 gpu.0.powerWatts‚222.618' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚74.20599999999999 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ûœÐ#h“‚(:Ý ¤êòÒü´Ú¤ disk.md0.out‚ 8.44140625& disk.nvme0n1.out‚10952.67724609375' proc.memory.availableMB‚ 323.4296875! disk.dm-1.out‚526.61279296875 network.sent‚420193 disk.nvme1n1.in‚ 114.96484375# disk.nvme1n1.out‚8821.158203125 disk.md0.in‚0" disk.dm-0.out‚19229.3505859375 disk.dm-1.in‚0 disk.nvme0c0n1.in‚ 1825.96875% disk.nvme1c1n1.out‚8821.158203125 disk.dm-0.in‚ 1940.93359375 disk.rbd0.in‚0( disk.nvme0c0n1.out‚10952.67724609375 disk.nvme0n1.in‚ 1825.96875$ memory_percent‚98.02594184875488 proc.memory.rssMB‚ 2790.4375* proc.memory.percent‚17.031478881835938 network.recv‚109230% disk./.usageGB‚1176.7584800720215 disk.rbd0.out‚0 cpu‚25.649663488979197 proc.cpu.threads‚17) disk./.usagePercent‚76.64930663227626" disk.nvme1c1n1.in‚ 114.96484375ê„’Õ”‚(:’ ¤êòÒ½ÿ¥ gpu.0.gpu‚55.0 gpu.0.memory‚9, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚50.0 gpu.0.powerWatts‚217.607' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚72.53566666666667 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0B‹Mþ0•+ _wandbruntime‚261  _runtime‚2617ÃýNÅ–¿ ' lr-AdamW/pg2‚0.00013889448413650842 trainer/global_step‚399 $ _timestamp‚1.784460587588282e+09 _runtime‚ 261.976716145 _step‚28 ' lr-AdamW/pg1‚0.00013889448413650842×;„É—Ã ' lr-AdamW/pg1‚0.00013889448413650842 ' lr-AdamW/pg2‚0.00013889448413650842 trainer/global_step‚399 $ _timestamp‚1.784460587588282e+09 _runtime‚ 261.976716145 _step‚28Xü‹0˜+ _wandbruntime‚261  _runtime‚261ÊG2Å{™©O ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 137190912 _runtime‚ 261.980838883 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 315549799424 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 315461319168 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚3940248 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106882647 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚22825872411648 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12023 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 HDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269121024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88480256 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚3055571 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 125429248 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚30394497038336 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚2899999 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚29734647817072 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 trainer/global_step‚399 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚22500443400704 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚22500349291008 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103824258 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚6995819 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9794 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2370606080 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 % _timestamp‚1.7844605875892792e+09 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚3950040 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 314657199484 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚30079052758528 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚3940248 HDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11952 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚30079052758528 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚2023697 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760ˆî'³'š­O G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269121024 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88480256 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚3055571 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 125429248 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚30394497038336 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚2899999 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚29734647817072 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 trainer/global_step‚399 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚22500443400704 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚22500349291008 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103824258 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚6995819 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚9794 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2370606080 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 % _timestamp‚1.7844605875892792e+09 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚3950040 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 314657199484 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚30079052758528 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚3940248 HDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚11952 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚30079052758528 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚2023697 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 137190912 _runtime‚ 261.980838883 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 315549799424 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 315461319168 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚3940248 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106882647 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚22825872411648 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12023 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 H6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚3950068 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 326324858880 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚7023076 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105526784 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚22882811877888 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103824370 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚3959874 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚29808921303104 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106882647 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚3950068 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚30124275212290 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12023 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚878467 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚7013268 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚22556581128704 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 326460766720 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 315440694196 _step‚30 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 316335522304 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚178 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 316247034880 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105526784 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚7023076 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚2907042 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 135907840 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267415751 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 316335522304 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 316247034880 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86785010 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚3063202 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚30470432420352 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚2907224 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚878289 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚30470537947136 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚30154202424832 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚30470537947136 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚30124379036660 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9806 O6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12023 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88487424 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301697536 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9806 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚29808938342464 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 315353909186 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269121024 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚22883041895424 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 230017536 % _timestamp‚1.7844605879887083e+09 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0±öpÏ !·M D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269121024 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚22883041895424 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 230017536 % _timestamp‚1.7844605879887083e+09 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚3063200 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚182 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚3950068 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 326324858880 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚7023076 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105526784 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚22882811877888 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103824370 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚3959874 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚29808921303104 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106882647 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚3950068 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚30124275212290 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12023 TIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚878467 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚7013268 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚22556581128704 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 326460766720 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 315440694196 _step‚30 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 316335522304 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚178 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 316247034880 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105526784 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚7023076 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚2907042 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 135907840 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267415751 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 316335522304 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 316247034880 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86785010 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚3063202 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚30470432420352 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚2907224 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 PEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚878289 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚30470537947136 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚30154202424832 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚30470537947136 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚30124379036660 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 H?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚9806 O6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12023 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88487424 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301697536 LCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚9806 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚29808938342464 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 315353909186T³‚0ž+ _wandbruntime‚262  _runtime‚262 _ÿŒÿŸù 2loss/SRqqt2/assignment_loss‚1.7341702914178114 1loss/FBt1/assignment_loss‚0.36418864260542555 epoch‚0 0loss/FRt2/assignment_loss‚3.9817192005984325 .loss/FBt2/assignment_loss‚0.24718228621664 /loss/FRt2/detection_loss‚1.4409437728575134 1loss/SRqqt1/detection_loss‚3.1562784024915826 /loss/FBt1/detection_loss‚15.274773971771323 % _timestamp‚1.7844605879896677e+09 /loss/FRt1/assignment_loss‚4.526941243884633 2loss/SRqqt1/assignment_loss‚1.8738714966497618 1loss/SRqqt2/detection_loss‚1.0691409201557993 trainer/global_step‚399 _runtime‚ 262.377768525 /loss/FRt1/detection_loss‚0.4109397952778435 .loss/FBt2/detection_loss‚5.674095873562544 %loss/total_loss‚39.75424589748931 _step‚31LúÔ ý epoch‚0 0loss/FRt2/assignment_loss‚3.9817192005984325 .loss/FBt2/assignment_loss‚0.24718228621664 /loss/FRt2/detection_loss‚1.4409437728575134 1loss/SRqqt1/detection_loss‚3.1562784024915826 /loss/FBt1/detection_loss‚15.274773971771323 % _timestamp‚1.7844605879896677e+09 /loss/FRt1/assignment_loss‚4.526941243884633 2loss/SRqqt1/assignment_loss‚1.8738714966497618 1loss/SRqqt2/detection_loss‚1.0691409201557993 trainer/global_step‚399 _runtime‚ 262.377768525 /loss/FRt1/detection_loss‚0.4109397952778435 .loss/FBt2/detection_loss‚5.674095873562544 %loss/total_loss‚39.75424589748931 _step‚31 2loss/SRqqt2/assignment_loss‚1.7341702914178114 1loss/FBt1/assignment_loss‚0.36418864260542555‡Àû²f¡‚(:Û ³êòÒÎáà¤) disk./.usagePercent‚76.47435856533252% disk./.usageGB‚1174.0725898742676 disk.rbd0.out‚0 disk.md0.out‚ 8.77734375$ memory_percent‚98.47879409790039! proc.memory.rssMB‚ 2864.328125 proc.cpu.threads‚17( disk.nvme0c0n1.out‚11112.53369140625 disk.nvme0n1.in‚ 1852.4375 disk.dm-0.in‚ 1982.19140625& proc.memory.availableMB‚ 249.234375 cpu‚25.7352237363033 network.sent‚423553 disk.rbd0.in‚0 disk.nvme0c0n1.in‚ 1852.4375& disk.nvme0n1.out‚11112.53369140625& disk.nvme1c1n1.out‚8913.7216796875 disk.nvme1n1.in‚ 129.75390625$ disk.nvme1n1.out‚8913.7216796875 disk.md0.in‚0 network.recv‚112772 disk.dm-1.in‚0 disk.dm-1.out‚ 527.009765625" disk.nvme1c1n1.in‚ 129.75390625# disk.dm-0.out‚19480.65771484375* proc.memory.percent‚17.482471466064453 ƒL¢‚(:’ ³êòÒ’ꦥ gpu.0.gpu‚44.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚49.0 gpu.0.powerWatts‚218.915' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚72.97166666666666 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0c!¤™J£bE "Purity/1FRt2SRqqt1FBt/event_purity"trainer/global_step2:JP“§ëöJ¤bE "Purity/0FRt1SRqqt2FBt/SRqqt_purity"trainer/global_step2:JPyÕ ÊJ¥bE "Purity/*FRt1SRqqt2FBt/event_purity"trainer/global_step2:JP,rt³H¦bC Purity/0FRt0SRqqt*FBt/FBt_purity"trainer/global_step2:JPa 1ÎH§bC Purity/2FRt1SRqqt0FBt/FRt_purity"trainer/global_step2:JP!ž;N¨bI &Purity/*FRt0SRqqt*FBt/event_proportion"trainer/global_step2:JPÃvÿ¾J©bE "Purity/*FRt0SRqqt0FBt/event_purity"trainer/global_step2:JPYÁ ÿNªbI &Purity/0FRt2SRqqt1FBt/event_proportion"trainer/global_step2:JPO¬/£J«bE "Purity/2FRt*SRqqt*FBt/event_purity"trainer/global_step2:JPB™ ~H¬bC Purity/*FRt1SRqqt1FBt/FBt_purity"trainer/global_step2:JP ªðbJ­bE "Purity/0FRt0SRqqt0FBt/event_purity"trainer/global_step2:JPŸ3ªÕH®bC Purity/0FRt1SRqqt*FBt/FBt_purity"trainer/global_step2:JP ²KH¯bC Purity/1FRt1SRqqt1FBt/FRt_purity"trainer/global_step2:JPG±Ä <°b7 particle/specificity"trainer/global_step2:JPå¢J±bE "Purity/0FRt2SRqqt1FBt/event_purity"trainer/global_step2:JPè{¨ðN²bI &Purity/1FRt1SRqqt*FBt/event_proportion"trainer/global_step2:JP|½”íH³bC Purity/2FRt1SRqqt*FBt/FBt_purity"trainer/global_step2:JP KÛH´bC Purity/2FRt*SRqqt2FBt/FRt_purity"trainer/global_step2:JPE@µb; particle/accuracy_2_of_5"trainer/global_step2:JPxñØJ¶bE "Purity/*FRt1SRqqt0FBt/SRqqt_purity"trainer/global_step2:JPáiŸN·bI &Purity/0FRt2SRqqt0FBt/event_proportion"trainer/global_step2:JP8˜ÔèH¸bC Purity/1FRt*SRqqt1FBt/FBt_purity"trainer/global_step2:JPêóqòN¹bI &Purity/2FRt*SRqqt2FBt/event_proportion"trainer/global_step2:JPÜ/ ¸JºbE "Purity/2FRt2SRqqt*FBt/SRqqt_purity"trainer/global_step2:JPHhš–H»bC Purity/2FRt0SRqqt1FBt/FBt_purity"trainer/global_step2:JP ±IAJ¼bE "Purity/*FRt1SRqqt0FBt/event_purity"trainer/global_step2:JPYæ‡N½bI &Purity/*FRt2SRqqt0FBt/event_proportion"trainer/global_step2:JPÞÛN¾bI &Purity/0FRt*SRqqt1FBt/event_proportion"trainer/global_step2:JP–~<@¿b; particle/accuracy_3_of_4"trainer/global_step2:JPðGLìNÀbI &Purity/*FRt0SRqqt0FBt/event_proportion"trainer/global_step2:JP×®×\HÁbC Purity/0FRt*SRqqt2FBt/FBt_purity"trainer/global_step2:JPH-ÂNÂbI &Purity/1FRt0SRqqt0FBt/event_proportion"trainer/global_step2:JPß6‹lHÃbC Purity/1FRt1SRqqt2FBt/FBt_purity"trainer/global_step2:JPä‚@fNÄbI &Purity/1FRt2SRqqt2FBt/event_proportion"trainer/global_step2:JP0g^LHÅbC Purity/2FRt*SRqqt*FBt/FBt_purity"trainer/global_step2:JPÉúvJÆbE "Purity/2FRt1SRqqt1FBt/event_purity"trainer/global_step2:JP¿±)$JÇbE "Purity/2FRt2SRqqt*FBt/event_purity"trainer/global_step2:JPýÍJÈbE "Purity/2FRt*SRqqt*FBt/SRqqt_purity"trainer/global_step2:JP¯ìfòNÉbI &Purity/2FRt2SRqqt*FBt/event_proportion"trainer/global_step2:JPÐ"ZDHÊbC Purity/2FRt2SRqqt*FBt/FRt_purity"trainer/global_step2:JP§ÇHËbC Purity/2FRt1SRqqt2FBt/FRt_purity"trainer/global_step2:JP6ÄÜNÌbI &Purity/1FRt2SRqqt0FBt/event_proportion"trainer/global_step2:JPƒÅ:@Íb; particle/accuracy_2_of_3"trainer/global_step2:JPRƉŒJÎbE "Purity/2FRt1SRqqt2FBt/SRqqt_purity"trainer/global_step2:JP8É@Ïb; particle/accuracy_4_of_5"trainer/global_step2:JPDîwJÐbE "Purity/*FRt2SRqqt*FBt/SRqqt_purity"trainer/global_step2:JPÌqÀ HÑbC Purity/*FRt2SRqqt1FBt/FRt_purity"trainer/global_step2:JPFÛpÒJÒbE "Purity/1FRt*SRqqt*FBt/SRqqt_purity"trainer/global_step2:JPï>ñJÓbE "Purity/1FRt*SRqqt2FBt/SRqqt_purity"trainer/global_step2:JP8]çÈHÔbC Purity/1FRt0SRqqt2FBt/FBt_purity"trainer/global_step2:JP¦þ±HÕbC Purity/2FRt*SRqqt0FBt/FRt_purity"trainer/global_step2:JP„ AH;Öb6 jet/accuracy_2_of_3"trainer/global_step2:JP¯k°¯H×bC Purity/*FRt*SRqqt1FBt/FRt_purity"trainer/global_step2:JPÞ9NØbI &Purity/2FRt1SRqqt2FBt/event_proportion"trainer/global_step2:JPÝnÀ@Ùb; particle/accuracy_3_of_3"trainer/global_step2:JPXÆŽƒJÚbE "Purity/*FRt*SRqqt1FBt/event_purity"trainer/global_step2:JP4qòHÛbC Purity/*FRt0SRqqt1FBt/FRt_purity"trainer/global_step2:JPª3®‘JÜbE "Purity/0FRt*SRqqt0FBt/SRqqt_purity"trainer/global_step2:JPV$JÝbE "Purity/0FRt0SRqqt2FBt/event_purity"trainer/global_step2:JP¤h…HÞbC Purity/1FRt2SRqqt1FBt/FBt_purity"trainer/global_step2:JP¾²I@ßb; particle/accuracy_2_of_4"trainer/global_step2:JP<ž|JàbE "Purity/0FRt*SRqqt2FBt/SRqqt_purity"trainer/global_step2:JP›a0ÄHábC Purity/*FRt0SRqqt2FBt/FRt_purity"trainer/global_step2:JPñ¶×÷NâbI &Purity/*FRt1SRqqt2FBt/event_proportion"trainer/global_step2:JPc¬ÆKNãbI &Purity/0FRt2SRqqt2FBt/event_proportion"trainer/global_step2:JPDsMiJäbE "Purity/1FRt1SRqqt2FBt/event_purity"trainer/global_step2:JP—9-gNåbI &Purity/2FRt2SRqqt2FBt/event_proportion"trainer/global_step2:JP1ŽtHæbC Purity/1FRt0SRqqt*FBt/FBt_purity"trainer/global_step2:JP~WEœHçbC Purity/1FRt1SRqqt0FBt/FRt_purity"trainer/global_step2:JPMq'NèbI &Purity/0FRt0SRqqt1FBt/event_proportion"trainer/global_step2:JP‰œ@éb; particle/accuracy_4_of_4"trainer/global_step2:JP[iLHêbC Purity/*FRt*SRqqt2FBt/FRt_purity"trainer/global_step2:JPô'ÛEJëbE "Purity/0FRt*SRqqt1FBt/event_purity"trainer/global_step2:JP;/¹JìbE "Purity/0FRt0SRqqt*FBt/event_purity"trainer/global_step2:JPÕE·œNíbI &Purity/0FRt1SRqqt0FBt/event_proportion"trainer/global_step2:JPκa•<îb7 particle/sensitivity"trainer/global_step2:JP#Ы JïbE "Purity/*FRt2SRqqt1FBt/SRqqt_purity"trainer/global_step2:JP?R“ƒNðbI &Purity/*FRt2SRqqt2FBt/event_proportion"trainer/global_step2:JPky¼HñbC Purity/0FRt*SRqqt*FBt/FBt_purity"trainer/global_step2:JP~c͆JòbE "Purity/1FRt0SRqqt0FBt/event_purity"trainer/global_step2:JPàYHóbC Purity/1FRt0SRqqt0FBt/FRt_purity"trainer/global_step2:JP8óÞÔHôbC Purity/1FRt1SRqqt*FBt/FBt_purity"trainer/global_step2:JP‘ó­ùJõbE "Purity/2FRt*SRqqt1FBt/event_purity"trainer/global_step2:JPÙŽvJöbE "Purity/2FRt1SRqqt0FBt/event_purity"trainer/global_step2:JPíÁpÍJ÷bE "Purity/1FRt*SRqqt0FBt/event_purity"trainer/global_step2:JPq.&øJøbE "Purity/1FRt1SRqqt1FBt/event_purity"trainer/global_step2:JPC!A¦HùbC Purity/1FRt2SRqqt*FBt/FRt_purity"trainer/global_step2:JP*RÿÖJúbE "Purity/2FRt0SRqqt*FBt/event_purity"trainer/global_step2:JPT:ù ;ûb6 jet/accuracy_1_of_4"trainer/global_step2:JPÞ—ž$8üb3 particle/f_score"trainer/global_step2:JP‰XNøHýbC Purity/2FRt*SRqqt2FBt/FBt_purity"trainer/global_step2:JPè·X$JþbE "Purity/0FRt*SRqqt*FBt/event_purity"trainer/global_step2:JPÒÀëCJÿbE "Purity/0FRt2SRqqt*FBt/SRqqt_purity"trainer/global_step2:JP”þ«N€bI &Purity/2FRt0SRqqt0FBt/event_proportion"trainer/global_step2:JP´R¾êJbE "Purity/2FRt*SRqqt2FBt/SRqqt_purity"trainer/global_step2:JP7îÆN‚bI &Purity/*FRt*SRqqt*FBt/event_proportion"trainer/global_step2:JP—J ™NƒbI &Purity/2FRt0SRqqt2FBt/event_proportion"trainer/global_step2:JPÁ‹fUH„bC Purity/2FRt1SRqqt1FBt/FRt_purity"trainer/global_step2:JP$¾ö¾J…bE "Purity/2FRt2SRqqt0FBt/SRqqt_purity"trainer/global_step2:JP·ö}#H†bC Purity/*FRt1SRqqt2FBt/FBt_purity"trainer/global_step2:JP*b dH‡bC Purity/0FRt2SRqqt1FBt/FBt_purity"trainer/global_step2:JP×5JˆbE "Purity/1FRt0SRqqt*FBt/event_purity"trainer/global_step2:JPw}àùJ‰bE "Purity/1FRt2SRqqt*FBt/event_purity"trainer/global_step2:JPÂnãNŠbI &Purity/2FRt0SRqqt1FBt/event_proportion"trainer/global_step2:JPáp÷éN‹bI &Purity/2FRt1SRqqt0FBt/event_proportion"trainer/global_step2:JP´ríENŒbI &Purity/2FRt2SRqqt0FBt/event_proportion"trainer/global_step2:JPî¶ ;b6 jet/accuracy_1_of_1"trainer/global_step2:JPd*.%JŽbE "Purity/2FRt1SRqqt2FBt/event_purity"trainer/global_step2:JP*^Ö}JbE "Purity/0FRt*SRqqt1FBt/SRqqt_purity"trainer/global_step2:JP”j^nNbI &Purity/0FRt0SRqqt0FBt/event_proportion"trainer/global_step2:JP´’™H‘bC Purity/1FRt*SRqqt2FBt/FBt_purity"trainer/global_step2:JPí½IJ’bE "Purity/2FRt1SRqqt*FBt/SRqqt_purity"trainer/global_step2:JP,¼î¤H“bC Purity/2FRt0SRqqt2FBt/FBt_purity"trainer/global_step2:JP±†s^J”bE "Purity/*FRt*SRqqt0FBt/event_purity"trainer/global_step2:JPÔ¶ágJ•bE "Purity/1FRt1SRqqt*FBt/SRqqt_purity"trainer/global_step2:JP}RÁN–bI &Purity/1FRt1SRqqt2FBt/event_proportion"trainer/global_step2:JPYQ«J—bE "Purity/2FRt*SRqqt0FBt/SRqqt_purity"trainer/global_step2:JPd[¿Ø;˜b6 jet/accuracy_4_of_4"trainer/global_step2:JPfnD‚@™b; particle/accuracy_3_of_5"trainer/global_step2:JPô ¿HšbC Purity/*FRt*SRqqt1FBt/FBt_purity"trainer/global_step2:JPÖtìAJ›bE "Purity/0FRt*SRqqt2FBt/event_purity"trainer/global_step2:JP‡®…eNœbI &Purity/0FRt0SRqqt2FBt/event_proportion"trainer/global_step2:JP¡ ç7JbE "Purity/1FRt1SRqqt1FBt/SRqqt_purity"trainer/global_step2:JP͇DJžbE "Purity/2FRt0SRqqt2FBt/event_purity"trainer/global_step2:JP…j©³NŸbI &Purity/*FRt*SRqqt0FBt/event_proportion"trainer/global_step2:JPˆ[\ÙJ bE "Purity/*FRt2SRqqt1FBt/event_purity"trainer/global_step2:JPáJH¡bC Purity/*FRt2SRqqt1FBt/FBt_purity"trainer/global_step2:JP¡ N¢bI &Purity/0FRt0SRqqt*FBt/event_proportion"trainer/global_step2:JPé) qJ£bE "Purity/2FRt*SRqqt1FBt/SRqqt_purity"trainer/global_step2:JP¹(H¤bC Purity/2FRt1SRqqt*FBt/FRt_purity"trainer/global_step2:JPõ;ÀsN¥bI &Purity/*FRt*SRqqt2FBt/event_proportion"trainer/global_step2:JP‡÷9‹N¦bI &Purity/1FRt*SRqqt*FBt/event_proportion"trainer/global_step2:JPlbµJ§bE "Purity/1FRt*SRqqt2FBt/event_purity"trainer/global_step2:JPÛ•ÄñJ¨bE "Purity/1FRt2SRqqt0FBt/event_purity"trainer/global_step2:JP=»ô@©b; particle/accuracy_1_of_5"trainer/global_step2:JPSê›HªbC Purity/*FRt0SRqqt1FBt/FBt_purity"trainer/global_step2:JP¥=ÂH«bC Purity/1FRt2SRqqt1FBt/FRt_purity"trainer/global_step2:JP ³ÏH¬bC Purity/*FRt1SRqqt0FBt/FRt_purity"trainer/global_step2:JPŠƒ§7J­bE "Purity/2FRt1SRqqt1FBt/SRqqt_purity"trainer/global_step2:JPê•@®b; particle/accuracy_1_of_2"trainer/global_step2:JPLÄ|'N¯bI &Purity/*FRt0SRqqt2FBt/event_proportion"trainer/global_step2:JP‰”J°bE "Purity/0FRt2SRqqt0FBt/event_purity"trainer/global_step2:JP¹¿N±bI &Purity/1FRt*SRqqt2FBt/event_proportion"trainer/global_step2:JPUÂÍ2N²bI &Purity/1FRt0SRqqt1FBt/event_proportion"trainer/global_step2:JP ² S;³b6 loss/val_total_loss"trainer/global_step2:JP®¼lßJ´bE "Purity/*FRt*SRqqt*FBt/SRqqt_purity"trainer/global_step2:JPPq—àJµbE "Purity/*FRt1SRqqt*FBt/SRqqt_purity"trainer/global_step2:JPl–lEH¶bC Purity/1FRt*SRqqt0FBt/FRt_purity"trainer/global_step2:JPˆ‰óJ·bE "Purity/1FRt1SRqqt2FBt/SRqqt_purity"trainer/global_step2:JPÓ'eXH¸bC Purity/2FRt0SRqqt0FBt/FRt_purity"trainer/global_step2:JPÌ{J¹bE "Purity/*FRt2SRqqt0FBt/SRqqt_purity"trainer/global_step2:JPÖnŒHºbC Purity/0FRt0SRqqt1FBt/FBt_purity"trainer/global_step2:JP ÂH»bC Purity/*FRt0SRqqt*FBt/FRt_purity"trainer/global_step2:JPŽ±ô;¼b6 jet/accuracy_3_of_4"trainer/global_step2:JPJÆbE "Purity/0FRt1SRqqt0FBt/SRqqt_purity"trainer/global_step2:JP|ËV~HÇbC Purity/2FRt1SRqqt1FBt/FBt_purity"trainer/global_step2:JP‡„?;NÈbI &Purity/2FRt2SRqqt1FBt/event_proportion"trainer/global_step2:JPĤ¨ÿHÉbC Purity/*FRt*SRqqt*FBt/FRt_purity"trainer/global_step2:JP" —}JÊbE "Purity/*FRt*SRqqt1FBt/SRqqt_purity"trainer/global_step2:JPcü(HËbC Purity/1FRt*SRqqt*FBt/FBt_purity"trainer/global_step2:JP1{c°HÌbC Purity/2FRt0SRqqt*FBt/FBt_purity"trainer/global_step2:JP]øéHÍbC Purity/1FRt*SRqqt2FBt/FRt_purity"trainer/global_step2:JP-U—HÎbC Purity/2FRt0SRqqt2FBt/FRt_purity"trainer/global_step2:JPsŒ¥dJÏbE "Purity/0FRt1SRqqt0FBt/event_purity"trainer/global_step2:JP'%ÎHÐbC Purity/2FRt*SRqqt1FBt/FBt_purity"trainer/global_step2:JP´ø<÷;Ñb6 jet/accuracy_2_of_5"trainer/global_step2:JPå&ÍßNÒbI &Purity/*FRt1SRqqt*FBt/event_proportion"trainer/global_step2:JP³ª½wHÓbC Purity/*FRt1SRqqt2FBt/FRt_purity"trainer/global_step2:JP W¡§NÔbI &Purity/*FRt2SRqqt1FBt/event_proportion"trainer/global_step2:JP ä©HÕbC Purity/1FRt0SRqqt*FBt/FRt_purity"trainer/global_step2:JP/S öJÖbE "Purity/1FRt1SRqqt0FBt/event_purity"trainer/global_step2:JPãRY²J×bE "Purity/0FRt1SRqqt*FBt/SRqqt_purity"trainer/global_step2:JPõŽÄOHØbC Purity/1FRt2SRqqt0FBt/FRt_purity"trainer/global_step2:JP‡— SJÙbE "Purity/1FRt2SRqqt1FBt/SRqqt_purity"trainer/global_step2:JP”z Q@Úb; particle/accuracy_1_of_1"trainer/global_step2:JP9h0 JÛbE "Purity/*FRt*SRqqt*FBt/event_purity"trainer/global_step2:JPö0Ä÷JÜbE "Purity/*FRt0SRqqt1FBt/event_purity"trainer/global_step2:JP.;=žJÝbE "Purity/*FRt1SRqqt*FBt/event_purity"trainer/global_step2:JPô°6|JÞbE "Purity/*FRt2SRqqt0FBt/event_purity"trainer/global_step2:JPür=JßbE "Purity/0FRt0SRqqt1FBt/event_purity"trainer/global_step2:JP{‡rBNàbI &Purity/1FRt2SRqqt1FBt/event_proportion"trainer/global_step2:JP™¹2NábI &Purity/2FRt*SRqqt1FBt/event_proportion"trainer/global_step2:JPÌ[Ø9âb4 particle/accuracy"trainer/global_step2:JPÄõ7_NãbI &Purity/1FRt1SRqqt1FBt/event_proportion"trainer/global_step2:JPÂÄÔJäbE "Purity/0FRt2SRqqt0FBt/SRqqt_purity"trainer/global_step2:JPP¨½—HåbC Purity/1FRt2SRqqt*FBt/FBt_purity"trainer/global_step2:JPncÏdJæbE "Purity/0FRt2SRqqt1FBt/SRqqt_purity"trainer/global_step2:JPu€géHçbC Purity/1FRt1SRqqt*FBt/FRt_purity"trainer/global_step2:JPÊ© NèbI &Purity/2FRt1SRqqt*FBt/event_proportion"trainer/global_step2:JPXèf@éb; particle/accuracy_2_of_2"trainer/global_step2:JPžŸë¸;êb6 jet/accuracy_5_of_5"trainer/global_step2:JPJÁ9ïNëbI &Purity/*FRt*SRqqt1FBt/event_proportion"trainer/global_step2:JPM|3&Purity/1FRt*SRqqt1FBt/event_proportion‚0.01643880270421505 7 Purity/1FRt*SRqqt1FBt/FRt_purity‚0.0715465098619461 >&Purity/1FRt2SRqqt*FBt/event_proportion‚0.02322823740541935 9"Purity/1FRt2SRqqt*FBt/SRqqt_purity‚0.1932869404554367 7 Purity/2FRt0SRqqt1FBt/FRt_purity‚0.0364583358168602 :"Purity/*FRt1SRqqt1FBt/event_purity‚0.09451345354318619 =&Purity/2FRt0SRqqt*FBt/event_proportion‚0.0208565853536129 particle/accuracy_1_of_4‚1 7 Purity/*FRt2SRqqt0FBt/FRt_purity‚0.1397695392370224 :"Purity/0FRt*SRqqt*FBt/SRqqt_purity‚0.18023209273815155 ;"Purity/0FRt2SRqqt*FBt/event_purity‚0.002993570873513818 & Purity/0FRt2SRqqt*FBt/FBt_purity‚1 =&Purity/2FRt*SRqqt0FBt/event_proportion‚0.0382486991584301 8 Purity/2FRt*SRqqt1FBt/FRt_purity‚0.07750000059604645 ("Purity/2FRt2SRqqt0FBt/event_purity‚0 ,jet/accuracy_3_of_3‚0.007131801452487707 9 Purity/*FRt*SRqqt0FBt/FRt_purity‚0.047450192272663116 =&Purity/*FRt1SRqqt0FBt/event_proportion‚0.3670479953289032 >&Purity/0FRt2SRqqt*FBt/event_proportion‚0.02457682229578495 =&Purity/1FRt*SRqqt0FBt/event_proportion‚0.6105840802192688 7 Purity/0FRt1SRqqt2FBt/FBt_purity‚0.8653846383094788 epoch‚0 9 Purity/*FRt0SRqqt0FBt/FRt_purity‚0.013273661956191063 9"Purity/1FRt*SRqqt0FBt/SRqqt_purity‚0.1969994753599167 :"Purity/1FRt0SRqqt2FBt/event_purity‚0.09489797055721283 =&Purity/1FRt0SRqqt*FBt/event_proportion‚0.4530552327632904 7 Purity/1FRt1SRqqt2FBt/FRt_purity‚0.2222222238779068 9"Purity/2FRt1SRqqt0FBt/SRqqt_purity‚0.1699497401714325 ( Purity/2FRt1SRqqt2FBt/FBt_purity‚0.5 7 Purity/*FRt*SRqqt*FBt/FBt_purity‚0.9883869886398315 9 Purity/1FRt*SRqqt*FBt/FRt_purity‚0.047703299671411514 =&Purity/1FRt1SRqqt0FBt/event_proportion‚0.1467633992433548 9 Purity/2FRt0SRqqt*FBt/FRt_purity‚0.007152996025979519 @&Purity/2FRt1SRqqt1FBt/event_proportion‚0.0008370535797439516 particle/accuracy_1_of_3‚1 & Purity/*FRt2SRqqt*FBt/FBt_purity‚1 9"Purity/0FRt1SRqqt1FBt/SRqqt_purity‚0.2738838791847229 ,jet/accuracy_2_of_2‚0.042534299194812775 % _timestamp‚1.7844606015449858e+09 8 Purity/*FRt1SRqqt*FBt/FRt_purity‚0.12432433664798737 9"Purity/*FRt1SRqqt1FBt/SRqqt_purity‚0.3158711791038513 7 Purity/0FRt0SRqqt2FBt/FBt_purity‚0.9710885882377625 8 Purity/1FRt0SRqqt2FBt/FRt_purity‚0.16116780042648315 7 Purity/*FRt0SRqqt*FBt/FBt_purity‚0.9887383580207825 <&Purity/*FRt2SRqqt*FBt/event_proportion‚0.054222472012043 :"Purity/1FRt*SRqqt1FBt/SRqqt_purity‚0.36639732122421265 jet/accuracy_1_of_5‚1 9 Purity/1FRt0SRqqt1FBt/FRt_purity‚0.044960107654333115 +jet/accuracy_2_of_4‚0.14174909889698029 6validation_average_jet_accuracy‚0.1358281970024109 jet/accuracy_3_of_5‚0 9"Purity/*FRt*SRqqt2FBt/SRqqt_purity‚0.0833333358168602 6 Purity/*FRt*SRqqt2FBt/FBt_purity‚0.967809796333313 :"Purity/0FRt*SRqqt0FBt/event_purity‚0.16002900898456573 ?&Purity/0FRt1SRqqt1FBt/event_proportion‚0.011555989272892475 <"Purity/2FRt0SRqqt0FBt/event_purity‚0.0009523809421807528 @&Purity/0FRt1SRqqt2FBt/event_proportion‚0.0004417782765813172 7 Purity/*FRt0SRqqt2FBt/FBt_purity‚0.9738560914993286 <"Purity/*FRt2SRqqt*FBt/event_purity‚0.0012849585618823767 ;"Purity/1FRt*SRqqt*FBt/event_purity‚0.011013013310730457 ("Purity/1FRt2SRqqt1FBt/event_purity‚0 :"Purity/0FRt1SRqqt2FBt/SRqqt_puritZ ¶¯ç(y‚0.03846153989434242 ("Purity/*FRt1SRqqt2FBt/event_purity‚0 7 Purity/0FRt0SRqqt*FBt/FBt_purity‚0.9869555234909058 8 Purity/2FRt1SRqqt0FBt/FRt_purity‚0.09174486249685287 =&Purity/*FRt0SRqqt*FBt/event_proportion‚0.5129278302192688 ;"Purity/*FRt0SRqqt0FBt/event_purity‚0.013394759967923164 @&Purity/0FRt2SRqqt1FBt/event_proportion‚0.0007207961170934141 <"Purity/2FRt*SRqqt*FBt/event_purity‚0.0004810004902537912 7 Purity/*FRt1SRqqt1FBt/FBt_purity‚0.9938595294952393 ("Purity/0FRt0SRqqt0FBt/event_purity‚1 7 Purity/0FRt1SRqqt*FBt/FBt_purity‚0.9832267761230469 8 Purity/1FRt1SRqqt1FBt/FRt_purity‚0.11517618596553802 +particle/specificity‚0.8331890106201172 ("Purity/0FRt2SRqqt1FBt/event_purity‚0 =&Purity/1FRt1SRqqt*FBt/event_proportion‚0.1534598171710968 7 Purity/2FRt1SRqqt*FBt/FBt_purity‚0.9736841917037964 + Purity/2FRt*SRqqt2FBt/FRt_purity‚0.0625 particle/accuracy_2_of_5‚1 9"Purity/*FRt1SRqqt0FBt/SRqqt_purity‚0.1859104335308075 >&Purity/0FRt2SRqqt0FBt/event_proportion‚0.02385602705180645 & Purity/1FRt*SRqqt1FBt/FBt_purity‚1 @&Purity/2FRt*SRqqt2FBt/event_proportion‚0.0002092633949359879 9"Purity/2FRt2SRqqt*FBt/SRqqt_purity‚0.1541280597448349 & Purity/2FRt0SRqqt1FBt/FBt_purity‚1 :"Purity/*FRt1SRqqt0FBt/event_purity‚0.10144298523664474 8&Purity/*FRt2SRqqt0FBt/event_proportion‚ 0.05322265625 =&Purity/0FRt*SRqqt1FBt/event_proportion‚0.0436662957072258 /particle/accuracy_3_of_4‚0.9978355169296265 =&Purity/*FRt0SRqqt0FBt/event_proportion‚0.4607747495174408 7 Purity/0FRt*SRqqt2FBt/FBt_purity‚0.9651787281036377 =&Purity/1FRt0SRqqt0FBt/event_proportion‚0.4408714771270752 7 Purity/1FRt1SRqqt2FBt/FBt_purity‚0.9444444179534912 ,&Purity/1FRt2SRqqt2FBt/event_proportion‚0 6 Purity/2FRt*SRqqt*FBt/FBt_purity‚0.987500011920929 ("Purity/2FRt1SRqqt1FBt/event_purity‚0 ("Purity/2FRt2SRqqt*FBt/event_purity‚0 :"Purity/2FRt*SRqqt*FBt/SRqqt_purity‚0.17433996498584747 ?&Purity/2FRt2SRqqt*FBt/event_proportion‚0.006417410913854837 7 Purity/2FRt2SRqqt*FBt/FRt_purity‚0.1452898532152176 & Purity/2FRt1SRqqt2FBt/FRt_purity‚0 8&Purity/1FRt2SRqqt0FBt/event_proportion‚ 0.02294921875 particle/accuracy_2_of_3‚1 ("Purity/2FRt1SRqqt2FBt/SRqqt_purity‚0 particle/accuracy_4_of_5‚0 7"Purity/*FRt2SRqqt*FBt/SRqqt_purity‚0.17713662981987 8 Purity/*FRt2SRqqt1FBt/FRt_purity‚0.10000000149011612 9"Purity/1FRt*SRqqt*FBt/SRqqt_purity‚0.2026413083076477 9"Purity/1FRt*SRqqt2FBt/SRqqt_purity‚0.1111111119389534 7 Purity/1FRt0SRqqt2FBt/FBt_purity‚0.9794973731040955 8 Purity/2FRt*SRqqt0FBt/FRt_purity‚0.05507499724626541 +jet/accuracy_2_of_3‚0.11134563386440277 8 Purity/*FRt*SRqqt1FBt/FRt_purity‚0.07266965508460999 @&Purity/2FRt1SRqqt2FBt/event_proportion‚2.325148852833081e-05 .particle/accuracy_3_of_3‚0.998457133769989 9"Purity/*FRt*SRqqt1FBt/event_purity‚0.5389914512634277 8 Purity/*FRt0SRqqt1FBt/FRt_purity‚0.04426819831132889 :"Purity/0FRt*SRqqt0FBt/SRqqt_purity‚0.17619262635707855 9"Purity/0FRt0SRqqt2FBt/event_purity‚0.9421773552894592 & Purity/1FRt2SRqqt1FBt/FBt_purity‚1 particle/accuracy_2_of_4‚1 :"Purity/0FRt*SRqqt2FBt/SRqqt_purity‚0.03846153989434242 8 Purity/*FRt0SRqqt2FBt/FRt_purity‚0.15347695350646973 @&Purity/*FRt1SRqqt2FBt/event_proportion‚0.0006975446594879031 ,&Purity/0FRt2SRqqt2FBt/event_proportion‚0 ("Purity/1FRt1SRqqt2FBt/event_purity‚0 ,&Purity/2FRt2SRqqt2FBt/event_proportion‚0 _runtime‚ 275.933843504 7 Purity/1FRt0SRqqt*FBt/FBt_purity‚0.9946109056472778 8 Purity/1FRt1SRqqt0FBt/FRt_purity‚0.13064229488372803 =&Purity/0FRt0SRqqt1FBt/event_proportion‚0.0313895083963871 .particle/accuracy_4_of_4‚0.933959424495697 7 Purity/*FRt*SRqqt2FBt/FRt_purity‚0.1547241061925888 9"Purity/0FRt*SRqqt1FBt/event_purity‚0.7525194883346558 9"Purity/0FRt0SRqqt*FBt/event_purity‚0.9844178557395935 =&Purity/0FRt1SRqqt0FBt/event_proportion‚0.2083565890789032 +particle/sensitivity‚0.5237545371055603 :"Purity/*FRt2SRqqt1FBt/SRqqt_purity‚0.21759259700775146 ,&Purity/*FRt2SRqqt2FBt/event_proportion‚0 7 Purity/0FRt*SRqqt*FBt/FBt_purity‚0.9860402941703796 ;"Purity/1FRt0SRqqt0FBt/event_purity‚0.013940060511231422 9 Purity/1FRt0SRqqt0FBt/FRt_purity‚0.013940060511231422 7 Purity/1FRt1SRqqt*FBt/FBt_purity‚0.9973545074462891 ("Purity/2FRt*SRqqt1FBt/event_purity‚0 ("Purity/2FRt1SRqqt0FBt/event_purity‚0 ;"Purity/1FRt*SRqqt0FBt/event_purity‚0.010293882340192795 ;"Purity/1FRt1SRqqt1FBt/event_purity‚0.023598138242959976 8 Purity/1FRt2SRqqt*FBt/FRt_purity‚0.13941392302513123 <"Purity/2FRt0SRqqt*FBt/event_purity‚0.0008818342466838658 )jet/accuracy_1_of_4‚0.575376570224762 (particle/f_score‚0.49093228578567505 + Purity/2FRt*SRqqt2FBt/FBt_purity‚0.9375 9"Purity/0FRt*SRqqt*FBt/event_purity‚0.2719821333885193 :"Purity/0FRt2SRqqt*FBt/SRqqt_purity‚0.16737020015716553 >&Purity/2FRt0SRqqt0FBt/event_proportion‚0.01990327425301075 ("Purity/2FRt*SRqqt2FBt/SRqqt_purity‚0 =&Purity/*FRt*SRqqt*FBt/event_proportion‚0.9537528157234192 A&Purity/2FRt0SRqqt2FBt/event_proportion‚0.00018601190822664648 7 Purity/2FRt1SRqqt1FBt/FRt_purity‚0.0972222238779068 9"Purity/2FRt2SRqqt0FBt/SRqqt_purity‚0.1541280597448349 7 Purity/*FRt1SRqqt2FBt/FBt_purity‚0.8888888955116272 & Purity/0FRt2SRqqt1FBt/FBt_purity‚1 ;"Purity/1FRt0SRqqt*FBt/event_purity‚0.014686841517686844 ("Purity/1FRt2SRqqt*FBt/event_purity‚0 @&Purity/2FRt0SRqqt1FBt/event_proportion‚0.0007672990905120969 ?&Purity/2FRt1SRqqt0FBt/event_proportion‚0.011928013525903225 ?&Purity/2FRt2SRqqt0FBt/event_proportion‚0.006417410913854837 *jet/accuracy_1_of_1‚0.1062929779291153 ("Purity/2FRt1SRqqt2FBt/event_purity‚0 9"Purity/0FRt*SRqqt1FBt/SRqqt_purity‚0.2606889307498932 =&Purity/0FRt0SRqqt0FBt/event_proportion‚0.0462472103536129 4 Purity/1FRt*SRqqt2FBt/FBt_purity‚0.9747354388237 :"Purity/2FRt1SRqqt*FBt/SRqqt_purity‚0.19270935654640198 & Purity/2FRt0SRqqt2FBt/FBt_purity‚1 :"Purity/*FRt*SRqqt0FBt/event_purity‚0.04932565242052078 :"Purity/1FRt1SRqqt*FBt/SRqqt_purity‚0.20563137531280518 @&Purity/1FRt1SRqqt2FBt/event_proportion‚0.0002325148816453293 9"Purity/2FRt*SRqqt0FBt/SRqqt_purity‚0.1632067710161209 jet/accuracy_4_of_4‚0 particle/accuracy_3_of_5‚1 7 Purity/*FRt*SRqqt1FBt/FBt_purity‚0.9955499768257141 9"Purity/0FRt*SRqqt2FBt/event_purity‚0.8867215514183044 ?&Purity/0FRt0SRqqt2FBt/event_proportion‚0.007626487873494625 9"Purity/1FRt1SRqqt1FBt/SRqqt_purity‚0.3722016513347626 ("Purity/2FRt0SRqqt2FBt/event_purity‚0 <&Purity/*FRt*SRqqt0FBt/event_proportion‚0.881045401096344 ("Purity/*FRt2SRqqt1FBt/event_purity‚0 & Purity/*FRt2SRqqt1FBt/FBt_purity‚1 =&Purity/0FRt0SRqqt*FBt/event_proportion‚0.0390159972012043 9"Purity/2FRt*SRqqt1FBt/SRqqt_purity‚0.5092592239379883 7 Purity/2FRt1SRqqt*FBt/FRt_purity‚0.0918579250574112 ?&Purity/*FRt*SRqqt2FBt/event_proportion‚0.010997953824698925 =&Purity/1FRt*SRqqt*FBt/event_proportion‚0.6297432780265808 :"Purity/1FRt*SRqqt2FBt/event_purity‚0.08866213262081146 ("Purity/1FRt2SRqqt0FBt/event_purity‚0 particle/accuracy_1_of_5‚1 7 Purity/*FRt0SRqqt1FBt/FBt_purity‚0.9960339069366455 8 Purity/1FRt2SRqqt1FBt/FRt_purity‚0.10000000149011612 8 Purity/*FRt1SRqqt0FBt/FRt_purity‚0.12499198317527771 9"Purity/2FRt1SRqqt1FBt/SRqqt_purity‚0.5092592239379883 particle/accuracy_1_of_2‚1 =&Purity/*FRt0SRqqt2FBt/event_proportion‚0.0103004090487957 ;"Purity/0FRt2SRqqt0FBt/event_purity‚0.003003152320161462 @&Purity/1FRt*SRqqt2FBt/event_proportion‚0.0027204242069274187 >&Purity/1FRt0SRqqt1FBt/event_proportion‚0.00969587080180645 )loss/val_total_loss‚29.62888711222143 :"Purity/*FRt*SRqqt*FBt/SRqqt_purity‚0.18882501125335693 9"Purity/*FRt1SRqqt*FBt/SRqqt_purity‚0.1921273022890091 9 Purity/1FRt*SRqqt0FBt/FRt_purity‚0.046494659036397934 9"Purity/1FRt1SRqqt2FBt/SRqqt_purity‚0.1111111119389534 9 Purity/2FRt0SRqqt0FBt/FRt_purity‚0.005278255790472031 7"Purity/*FRt2SRqqt0FBt/SRqqt_purity‚0.17662313580513 7 Purity/0FRt0SRqqt1FBt/FBt_purity‚0.9945588707923889 9 Purity/*FRt0SRqqt*FBt/FRt_purity‚0.014765909872949123 -jet/accuracy_3_of_4‚0.0029761905316263437 ("Purity/2FRt*SRqqt2FBt/event_purity‚0 jet/accuracy_4_of_5‚0 7 Purity/*FRt1SRqqt*FBt/FBt_purity‚0.9866877794265747 =&Purity/*FRt1SRqqt1FBt/event_proportion‚0.0188569575548172 9"Purity/0FRt1SRqqt*FBt/event_purity‚0.1759653389453888 =&Purity/0FRt1SRqqt*FBt/event_proportion‚0.2203543484210968 7 Purity/0FRt1SRqqt1FBt/FBt_purity‚0.9903722405433655 :"Purity/1FRt1SRqqt0FBt/SRqqt_purity‚0.19867996871471405 7 Purity/*FRt2SRqqt*FBt/FRt_purity‚0.1392184942960739 9"Purity/0FRt1SRqqt0FBt/SRqqt_purity‚0.1780555695295334 & Purity/2FRt1SRqqt1FBt/FBt_purity‚1 ,&Purity/2FRt2SRqqt1FBt/event_proportion‚0 9 Purity/*FRt*SRqqt*FBt/FRt_purity‚0.048604488372802734 9"Purity/*FRt*SRqqt1FBt/SRqqt_purity‚0.3038422465324402 7 Purity/1FRt*SRqqt*FBt/FBt_purity‚0.9954163432121277 & Purity/2FRt0SRqqt*FBt/FBt_purity‚1 7 Purity/1FRt*SRqqt2FBt/FRt_purity‚0.1616780012845993 7 Purity/2FRt0SRqqt2FBt/FRt_purity‚0.0714285746216774 9"Purity/0FRt1SRqqt0FBt/event_purity‚0.1780555695295334 & Purity/2FRt*SRqqt1FBt/FBt_purity‚1 jet/accuracy_2_of_5‚0 trainer/global_step‚399 =&Purity/*FRt1SRqqt*FBt/event_proportion‚0.3866024911403656 8 Purity/*FRt1SRqqt2FBt/FRt_purity‚0.20000000298023224 @&Purity/*FRt2SRqqt1FBt/event_proportion‚0.0009998140158131719 9 Purity/1FRt0SRqqt*FBt/FRt_purity‚0.015409157611429691 <"Purity/1FRt1SRqqt0FBt/event_purity‚0.0009253207826986909 9"Purity/0FRt1SRqqt*FBt/SRqqt_purity‚0.1830442100763321 8 Purity/1FRt2SRqqt0FBt/FRt_purity‚0.14055564999580383 9"Purity/1FRt2SRqqt1FBt/SRqqt_purity‚0.2750000059604645 particle/accuracy_1_of_1‚1 9"Purity/*FRt*SRqqt*FBt/event_purity‚0.0882725864648819 9"Purity/*FRt0SRqqt1FBt/event_purity‚0.7527325749397278 :"Purity/*FRt1SRqqt*FBt/event_purity‚0.10099150240421295 <"Purity/*FRt2SRqqt0FBt/event_purity‚0.0012997755547985435 9"Purity/0FRt0SRqqt1FBt/event_purity‚0.9945588707923889 A&Purity/1FRt2SRqqt1FBt/event_proportion‚0.00027901786961592734 @&Purity/2FRt*SRqqt1FBt/event_proportion‚0.0016043527284637094 (particle/accuracy‚0.7667332887649536 @&Purity/1FRt1SRqqt1FBt/event_proportion‚0.0064639137126505375€l©ò=Åì{ 9"Purity/*FRt1SRqqt0FBt/SRqqt_purity‚0.1859104335308075 >&Purity/0FRt2SRqqt0FBt/event_proportion‚0.02385602705180645 & Purity/1FRt*SRqqt1FBt/FBt_purity‚1 @&Purity/2FRt*SRqqt2FBt/event_proportion‚0.0002092633949359879 9"Purity/2FRt2SRqqt*FBt/SRqqt_purity‚0.1541280597448349 & Purity/2FRt0SRqqt1FBt/FBt_purity‚1 :"Purity/*FRt1SRqqt0FBt/event_purity‚0.10144298523664474 8&Purity/*FRt2SRqqt0FBt/event_proportion‚ 0.05322265625 =&Purity/0FRt*SRqqt1FBt/event_proportion‚0.0436662957072258 /particle/accuracy_3_of_4‚0.9978355169296265 =&Purity/*FRt0SRqqt0FBt/event_proportion‚0.4607747495174408 7 Purity/0FRt*SRqqt2FBt/FBt_purity‚0.9651787281036377 =&Purity/1FRt0SRqqt0FBt/event_proportion‚0.4408714771270752 7 Purity/1FRt1SRqqt2FBt/FBt_purity‚0.9444444179534912 ,&Purity/1FRt2SRqqt2FBt/event_proportion‚0 6 Purity/2FRt*SRqqt*FBt/FBt_purity‚0.987500011920929 ("Purity/2FRt1SRqqt1FBt/event_purity‚0 ("Purity/2FRt2SRqqt*FBt/event_purity‚0 :"Purity/2FRt*SRqqt*FBt/SRqqt_purity‚0.17433996498584747 ?&Purity/2FRt2SRqqt*FBt/event_proportion‚0.006417410913854837 7 Purity/2FRt2SRqqt*FBt/FRt_purity‚0.1452898532152176 & Purity/2FRt1SRqqt2FBt/FRt_purity‚0 8&Purity/1FRt2SRqqt0FBt/event_proportion‚ 0.02294921875 particle/accuracy_2_of_3‚1 ("Purity/2FRt1SRqqt2FBt/SRqqt_purity‚0 particle/accuracy_4_of_5‚0 7"Purity/*FRt2SRqqt*FBt/SRqqt_purity‚0.17713662981987 8 Purity/*FRt2SRqqt1FBt/FRt_purity‚0.10000000149011612 9"Purity/1FRt*SRqqt*FBt/SRqqt_purity‚0.2026413083076477 9"Purity/1FRt*SRqqt2FBt/SRqqt_purity‚0.1111111119389534 7 Purity/1FRt0SRqqt2FBt/FBt_purity‚0.9794973731040955 8 Purity/2FRt*SRqqt0FBt/FRt_purity‚0.05507499724626541 +jet/accuracy_2_of_3‚0.11134563386440277 8 Purity/*FRt*SRqqt1FBt/FRt_purity‚0.07266965508460999 @&Purity/2FRt1SRqqt2FBt/event_proportion‚2.325148852833081e-05 .particle/accuracy_3_of_3‚0.998457133769989 9"Purity/*FRt*SRqqt1FBt/event_purity‚0.5389914512634277 8 Purity/*FRt0SRqqt1FBt/FRt_purity‚0.04426819831132889 :"Purity/0FRt*SRqqt0FBt/SRqqt_purity‚0.17619262635707855 9"Purity/0FRt0SRqqt2FBt/event_purity‚0.9421773552894592 & Purity/1FRt2SRqqt1FBt/FBt_purity‚1 particle/accuracy_2_of_4‚1 :"Purity/0FRt*SRqqt2FBt/SRqqt_purity‚0.03846153989434242 8 Purity/*FRt0SRqqt2FBt/FRt_purity‚0.15347695350646973 @&Purity/*FRt1SRqqt2FBt/event_proportion‚0.0006975446594879031 ,&Purity/0FRt2SRqqt2FBt/event_proportion‚0 ("Purity/1FRt1SRqqt2FBt/event_purity‚0 ,&Purity/2FRt2SRqqt2FBt/event_proportion‚0 _runtime‚ 275.933843504 7 Purity/1FRt0SRqqt*FBt/FBt_purity‚0.9946109056472778 8 Purity/1FRt1SRqqt0FBt/FRt_purity‚0.13064229488372803 =&Purity/0FRt0SRqqt1FBt/event_proportion‚0.0313895083963871 .particle/accuracy_4_of_4‚0.933959424495697 7 Purity/*FRt*SRqqt2FBt/FRt_purity‚0.1547241061925888 9"Purity/0FRt*SRqqt1FBt/event_purity‚0.7525194883346558 9"Purity/0FRt0SRqqt*FBt/event_purity‚0.9844178557395935 =&Purity/0FRt1SRqqt0FBt/event_proportion‚0.2083565890789032 +particle/sensitivity‚0.5237545371055603 :"Purity/*FRt2SRqqt1FBt/SRqqt_purity‚0.21759259700775146 ,&Purity/*FRt2SRqqt2FBt/event_proportion‚0 7 Purity/0FRt*SRqqt*FBt/FBt_purity‚0.9860402941703796 ;"Purity/1FRt0SRqqt0FBt/event_purity‚0.013940060511231422 9 Purity/1FRt0SRqqt0FBt/FRt_purity‚0.013940060511231422 7 Purity/1FRt1SRqqt*FBt/FBt_purity‚0.9973545074462891 ("Purity/2FRt*SRqqt1FBt/event_purity‚0 ("Purity/2FRt1SRqqt0FBt/event_purity‚0 ;"Purity/1FRt*SRqqt0FBt/event_purity‚0.010293882340192795 ;"Purity/1FRt1SRqqt1FBt/event_purity‚0.023598138242959976 8 Purity/1FRt2SRqqt*FBt/FRt_purity‚0.13941392302513123 <"Purity/2FRt0SRqqt*FBt/event_purity‚0.0008818342466838658 )jet/accuracy_1_of_4‚0.575376570224762 (particle/f_score‚0.49093228578567505 + Purity/2FRt*SRqqt2FBt/FBt_purity‚0.9375 9"Purity/0FRt*SRqqt*FBt/event_purity‚0.2719821333885193 :"Purity/0FRt2SRqqt*FBt/SRqqt_purity‚0.16737020015716553 >&Purity/2FRt0SRqqt0FBt/event_proportion‚0.01990327425301075 ("Purity/2FRt*SRqqt2FBt/SRqqt_purity‚0 =&Purity/*FRt*SRqqt*FBt/event_proportion‚0.9537528157234192 A&Purity/2FRt0SRqqt2FBt/event_proportion‚0.00018601190822664648 7 Purity/2FRt1SRqqt1FBt/FRt_purity‚0.0972222238779068 9"Purity/2FRt2SRqqt0FBt/SRqqt_purity‚0.1541280597448349 7 Purity/*FRt1SRqqt2FBt/FBt_purity‚0.8888888955116272 & Purity/0FRt2SRqqt1FBt/FBt_purity‚1 ;"Purity/1FRt0SRqqt*FBt/event_purity‚0.014686841517686844 ("Purity/1FRt2SRqqt*FBt/event_purity‚0 @&Purity/2FRt0SRqqt1FBt/event_proportion‚0.0007672990905120969 ?&Purity/2FRt1SRqqt0FBt/event_proportion‚0.011928013525903225 ?&Purity/2FRt2SRqqt0FBt/event_proportion‚0.006417410913854837 *jet/accuracy_1_of_1‚0.1062929779291153 ("Purity/2FRt1SRqqt2FBt/event_purity‚0 9"Purity/0FRt*SRqqt1FBt/SRqqt_purity‚0.2606889307498932 =&Purity/0FRt0SRqqt0FBt/event_proportion‚0.0462472103536129 4 Purity/1FRt*SRqqt2FBt/FBt_purity‚0.9747354388237 :"Purity/2FRt1SRqqt*FBt/SRqqt_purity‚0.19270935654640198 & Purity/2FRt0SRqqt2FBt/FBt_purity‚1 :"Purity/*FRt*SRqqt0FBt/event_purity‚0.04932565242052078 :"Purity/1FRt1SRqqt*FBt/SRqqt_purity‚0.20563137531280518 @&Purity/1FRt1SRqqt2FBt/event_proportion‚0.0002325148816453293 9"Purity/2FRt*SRqqt0FBt/SRqqt_purity‚0.1632067710161209 jet/accuracy_4_of_4‚0 particle/accuracy_3_of_5‚1 7 Purity/*FRt*SRqqt1FBt/FBt_purity‚0.9955499768257141 9"Purity/0FRt*SRqqt2FBt/event_purity‚0.8867215514183044 ?&Purity/0FRt0SRqqt2FBt/event_proportion‚0.007626487873494625 9"Purity/1FRt1SRqqt1FBt/SRqqt_purity‚0.3722016513347626 ("Purity/2FRt0SRqqt2FBt/event_purity‚0 <&Purity/*FRt*SRqqt0FBt/event_proportion‚0.881045401096344 ("Purity/*FRt2SRqqt1FBt/event_purity‚0 & Purity/*FRt2SRqqt1FBt/FBt_purity‚1 =&Purity/0FRt0SRqqt*FBt/event_proportion‚0.0390159972012043 9"Purity/2FRt*SRqqt1FBt/SRqqt_purity‚0.5092592239379883 7 Purity/2FRt1SRqqt*FBt/FRt_purity‚0.0918579250574112 ?&Purity/*FRt*SRqqt2FBt/event_proportion‚0.010997953824698925 =&Purity/1FRt*SRqqt*FBt/event_proportion‚0.6297432780265808 :"Purity/1FRt*SRqqt2FBt/event_purity‚0.08866213262081146 ("Purity/1FRt2SRqqt0FBt/event_purity‚0 particle/accuracy_1_of_5‚1 7 Purity/*FRt0SRqqt1FBt/FBt_purity‚0.9960339069366455 8 Purity/1FRt2SRqqt1FBt/FRt_purity‚0.10000000149011612 8 Purity/*FRt1SRqqt0FBt/FRt_purity‚0.12499198317527771 9"Purity/2FRt1SRqqt1FBt/SRqqt_purity‚0.5092592239379883 particle/accuracy_1_of_2‚1 =&Purity/*FRt0SRqqt2FBt/event_proportion‚0.0103004090487957 ;"Purity/0FRt2SRqqt0FBt/event_purity‚0.003003152320161462 @&Purity/1FRt*SRqqt2FBt/event_proportion‚0.0027204242069274187 >&Purity/1FRt0SRqqt1FBt/event_proportion‚0.00969587080180645 )loss/val_total_loss‚29.62888711222143 :"Purity/*FRt*SRqqt*FBt/SRqqt_purity‚0.18882501125335693 9"Purity/*FRt1SRqqt*FBt/SRqqt_purity‚0.1921273022890091 9 Purity/1FRt*SRqqt0FBt/FRt_purity‚0.046494659036397934 9"Purity/1FRt1SRqqt2FBt/SRqqt_purity‚0.1111111119389534 9 Purity/2FRt0SRqqt0FBt/FRt_purity‚0.005278255790472031 7"Purity/*FRt2SRqqt0FBt/SRqqt_purity‚0.17662313580513 7 Purity/0FRt0SRqqt1FBt/FBt_purity‚0.9945588707923889 9 Purity/*FRt0SRqqt*FBt/FRt_purity‚0.014765909872949123 -jet/accuracy_3_of_4‚0.0029761905316263437 ("Purity/2FRt*SRqqt2FBt/event_purity‚0 jet/accuracy_4_of_5‚0 7 Purity/*FRt1SRqqt*FBt/FBt_purity‚0.9866877794265747 =&Purity/*FRt1SRqqt1FBt/event_proportion‚0.0188569575548172 9"Purity/0FRt1SRqqt*FBt/event_purity‚0.1759653389453888 =&Purity/0FRt1SRqqt*FBt/event_proportion‚0.2203543484210968 7 Purity/0FRt1SRqqt1FBt/FBt_purity‚0.9903722405433655 :"Purity/1FRt1SRqqt0FBt/SRqqt_purity‚0.19867996871471405 7 Purity/*FRt2SRqqt*FBt/FRt_purity‚0.1392184942960739 9"Purity/0FRt1SRqqt0FBt/SRqqt_purity‚0.1780555695295334 & Purity/2FRt1SRqqt1FBt/FBt_purity‚1 ,&Purity/2FRt2SRqqt1FBt/event_proportion‚0 9 Purity/*FRt*SRqqt*FBt/FRt_purity‚0.048604488372802734 9"Purity/*FRt*SRqqt1FBt/SRqqt_purity‚0.3038422465324402 7 Purity/1FRt*SRqqt*FBt/FBt_purity‚0.9954163432121277 & Purity/2FRt0SRqqt*FBt/FBt_purity‚1 7 Purity/1FRt*SRqqt2FBt/FRt_purity‚0.1616780012845993 7 Purity/2FRt0SRqqt2FBt/FRt_purity‚0.0714285746216774 9"Purity/0FRt1SRqqt0FBt/event_purity‚0.1780555695295334 & Purity/2FRt*SRqqt1FBt/FBt_purity‚1 jet/accuracy_2_of_5‚0 trainer/global_step‚399 =&Purity/*FRt1SRqqt*FBt/event_proportion‚0.3866024911403656 8 Purity/*FRt1SRqqt2FBt/FRt_purity‚0.20000000298023224 @&Purity/*FRt2SRqqt1FBt/event_proportion‚0.0009998140158131719 9 Purity/1FRt0SRqqt*FBt/FRt_purity‚0.015409157611429691 <"Purity/1FRt1SRqqt0FBt/event_purity‚0.0009253207826986909 9"Purity/0FRt1SRqqt*FBt/SRqqt_purity‚0.1830442100763321 8 Purity/1FRt2SRqqt0FBt/FRt_purity‚0.14055564999580383 9"Purity/1FRt2SRqqt1FBt/SRqqt_purity‚0.2750000059604645 particle/accuracy_1_of_1‚1 9"Purity/*FRt*SRqqt*FBt/event_purity‚0.0882725864648819 9"Purity/*FRt0SRqqt1FBt/event_purity‚0.7527325749397278 :"Purity/*FRt1SRqqt*FBt/event_purity‚0.10099150240421295 <"Purity/*FRt2SRqqt0FBt/event_purity‚0.0012997755547985435 9"Purity/0FRt0SRqqt1FBt/event_purity‚0.9945588707923889 A&Purity/1FRt2SRqqt1FBt/event_proportion‚0.00027901786961592734 @&Purity/2FRt*SRqqt1FBt/event_proportion‚0.0016043527284637094 (particle/accuracy‚0.7667332887649536 @&Purity/1FRt1SRqqt1FBt/event_proportion‚0.0064639137126505375 :"Purity/0FRt2SRqqt0FBt/SRqqt_purity‚0.16752807796001434 & Purity/1FRt2SRqqt*FBt/FBt_purity‚1 0"Purity/0FRt2SRqqt1FBt/SRqqt_purity‚ 0.1640625 7 Purity/1FRt1SRqqt*FBt/FRt_purity‚0.1300354301929474 =&Purity/2FRt1SRqqt*FBt/event_proportion‚0.0127883180975914 particle/accuracy_2_of_2‚1 jet/accuracy_5_of_5‚0 =&Purity/*FRt*SRqqt1FBt/event_proportion‚0.0617094486951828 ?&Purity/0FRt*SRqqt2FBt/event_proportion‚0.008068266324698925 *jet/accuracy_1_of_3‚0.5556546449661255 ("Purity/0FRt1SRqqt2FBt/event_purity‚0 =&Purity/*FRt0SRqqt1FBt/event_proportion‚0.0418526791036129 9"Purity/*FRt0SRqqt2FBt/event_purity‚0.7185179591178894 9"Purity/*FRt1SRqqt2FBt/SRqqt_purity‚0.0833333358168602 =&Purity/0FRt*SRqqt0FBt/event_proportion‚0.2322126179933548 & Purity/1FRt0SRqqt1FBt/FBt_purity‚1 :"Purity/1FRt2SRqqt0FBt/SRqqt_purity‚0.19173438847064972 =&Purity/0FRt*SRqqt*FBt/event_proportion‚0.2839471697807312 :"Purity/1FRt0SRqqt1FBt/event_purity‚0.02919628843665123 9 Purity/2FRt*SRqqt*FBt/FRt_purity‚0.055622827261686325 9"Purity/*FRt*SRqqt2FBt/event_purity‚0.6717153787612915 ("Purity/2FRt0SRqqt1FBt/event_purity‚0 particle/accuracy_5_of_5‚0 9"Purity/*FRt*SRqqt0FBt/SRqqt_purity‚0.1838449239730835 :"Purity/*FRt0SRqqt*FBt/event_purity‚0.08778196573257446 8 Purity/*FRt1SRqqt1FBt/FRt_purity‚0.11194749176502228 7 Purity/0FRt*SRqqt1FBt/FBt_purity‚0.9936846494674683 9"Purity/0FRt1SRqqt1FBt/event_purity‚0.1432967633008957 @&Purity/1FRt0SRqqt2FBt/event_proportion‚0.0024879092816263437 & Purity/1FRt1SRqqt1FBt/FBt_purity‚1 <&Purity/2FRt*SRqqt*FBt/event_proportion‚0.040062315762043 ;"Purity/1FRt1SRqqt*FBt/event_purity‚0.001776272663846612 <"Purity/2FRt*SRqqt0FBt/event_purity‚0.0005175983533263206 ("Purity/2FRt1SRqqt*FBt/event_purity‚0 7 Purity/2FRt2SRqqt0FBt/FRt_purity‚0.1452898532152176 *jet/accuracy_1_of_2‚0.3724028170108795 _step‚32 ;"Purity/1FRt*SRqqt1FBt/event_purity‚0.026519151404500008 >&Purity/1FRt*SRqqt1FBt/event_proportion‚0.01643880270421505 7 Purity/1FRt*SRqqt1FBt/FRt_purity‚0.0715465098619461 >&Purity/1FRt2SRqqt*FBt/event_proportion‚0.02322823740541935 9"Purity/1FRt2SRqqt*FBt/SRqqt_purity‚0.1932869404554367 7 Purity/2FRt0SRqqt1FBt/FRt_purity‚0.0364583358168602 :"Purity/*FRt1SRqqt1FBt/event_purity‚0.09451345354318619 =&Purity/2FRt0SRqqt*FBt/event_proportion‚0.0208565853536129 particle/accuracy_1_of_4‚1 7 Purity/*FRt2SRqqt0FBt/FRt_purity‚0.1397695392370224 :"Purity/0FRt*SRqqt*FBt/SRqqt_purity‚0.18023209273815155 ;"Purity/0FRt2SRqqt*FBt/event_purity‚0.002993570873513818 & Purity/0FRt2SRqqt*FBt/FBt_purity‚1 =&Purity/2FRt*SRqqt0FBt/event_proportion‚0.0382486991584301 8 Purity/2FRt*SRqqt1FBt/FRt_purity‚0.07750000059604645 ("Purity/2FRt2SRqqt0FBt/event_purity‚0 ,jet/accuracy_3_of_3‚0.007131801452487707 9 Purity/*FRt*SRqqt0FBt/FRt_purity‚0.047450192272663116 =&Purity/*FRt1SRqqt0FBt/event_proportion‚0.3670479953289032 >&Purity/0FRt2SRqqt*FBt/event_proportion‚0.02457682229578495 =&Purity/1FRt*SRqqt0FBt/event_proportion‚0.6105840802192688 7 Purity/0FRt1SRqqt2FBt/FBt_purity‚0.8653846383094788 epoch‚0 9 Purity/*FRt0SRqqt0FBt/FRt_purity‚0.013273661956191063 9"Purity/1FRt*SRqqt0FBt/SRqqt_purity‚0.1969994753599167 :"Purity/1FRt0SRqqt2FBt/event_purity‚0.09489797055721283 =&Purity/1FRt0SRqqt*FBt/event_proportion‚0.4530552327632904 7 Purity/1FRt1SRqqt2FBt/FRt_purity‚0.2222222238779068 9"Purity/2FRt1SRqqt0FBt/SRqqt_purity‚0.1699497401714325 ( Purity/2FRt1SRqqt2FBt/FBt_purity‚0.5 7 Purity/*FRt*SRqqt*FBt/FBt_purity‚0.9883869886398315 9 Purity/1FRt*SRqqt*FBt/FRt_purity‚0.047703299671411514 =&Purity/1FRt1SRqqt0FBt/event_proportion‚0.1467633992433548 9 Purity/2FRt0SRqqt*FBt/FRt_purity‚0.007152996025979519 @&Purity/2FRt1SRqqt1FBt/event_proportion‚0.0008370535797439516 particle/accuracy_1_of_3‚1 & Purity/*FRt2SRqqt*FBt/FBt_purity‚1 9"Purity/0FRt1SRqqt1FBt/SRqqt_purity‚0.2738838791847229 ,jet/accuracy_2_of_2‚0.042534299194812775 % _timestamp‚1.7844606015449858e+09 8 Purity/*FRt1SRqqt*FBt/FRt_purity‚0.12432433664798737 9"Purity/*FRt1SRqqt1FBt/SRqqt_purity‚0.3158711791038513 7 Purity/0FRt0SRqqt2FBt/FBt_purity‚0.9710885882377625 8 Purity/1FRt0SRqqt2FBt/FRt_purity‚0.16116780042648315 7 Purity/*FRt0SRqqt*FBt/FBt_purity‚0.9887383580207825 <&Purity/*FRt2SRqqt*FBt/event_proportion‚0.054222472012043 :"Purity/1FRt*SRqqt1FBt/SRqqt_purity‚0.36639732122421265 jet/accuracy_1_of_5‚1 9 Purity/1FRt0SRqqt1FBt/FRt_purity‚0.044960107654333115 +jet/accuracy_2_of_4‚0.14174909889698029 6validation_average_jet_accuracy‚0.1358281970024109 jet/accuracy_3_of_5‚0 9"Purity/*FRt*SRqqt2FBt/SRqqt_purity‚0.0833333358168602 6 Purity/*FRt*SRqqt2FBt/FBt_purity‚0.967809796333313 :"Purity/0FRt*SRqqt0FBt/event_purity‚0.16002900898456573 ?&Purity/0FRt1SRqqt1FBt/event_proportion‚0.011555989272892475 <"Purity/2FRt0SRqqt0FBt/event_purity‚0.0009523809421807528 @&Purity/0FRt1SRqqt2FBt/event_proportion‚0.0004417782765813172 7 Purity/*FRt0SRqqt2FBt/FBt_purity‚0.9738560914993286 <"Purity/*FRt2SRqqt*FBt/event_purity‚0.0012849585618823767 ;"Purity/1FRt*SRqqt*FBt/event_purity‚0.011013013310730457 ("Purity/1FRt2SRqqt1FBt/event_purity‚0 :"Purity/0FRt1SRqqt2FBt/SRqqt_purity‚0.03846153989434242 ("Purity/*FRt1SRqqt2FBt/event_purity‚0 7 Purity/0FRt0SRqqt*FBt/FBt_purity‚0.9869555234909058 8 Purity/2FRt1SRqqt0FBt/FRt_purity‚0.09174486249685287 =&Purity/*FRt0SRqqt*FBt/event_proportion‚0.5129278302192688 ;"Purity/*FRt0SRqqt0FBt/event_purity‚0.013394759967923164 @&Purity/0FRt2SRqqt1FBt/event_proportion‚0.0007207961170934141 <"Purity/2FRt*SRqqt*FBt/event_purity‚0.0004810004902537912 7 Purity/*FRt1SRqqt1FBt/FBt_purity‚0.9938595294952393 ("Purity/0FRt0SRqqt0FBt/event_purity‚1 7 Purity/0FRt1SRqqt*FBt/FBt_purity‚0.9832267761230469 8 Purity/1FRt1SRqqt1FBt/FRt_purity‚0.11517618596553802 +particle/specificity‚0.8331890106201172 ("Purity/0FRt2SRqqt1FBt/event_purity‚0 =&Purity/1FRt1SRqqt*FBt/event_proportion‚0.1534598171710968 7 Purity/2FRt1SRqqt*FBt/FBt_purity‚0.9736841917037964 + Purity/2FRt*SRqqt2FBt/FRt_purity‚0.0625 particle/accuracy_2_of_5‚1 ôivXiÆ‚(:Þ ÂêòÒí˜È¤) disk./.usagePercent‚76.43859669471522 disk.md0.out‚ 9.28515625 disk.rbd0.in‚0 proc.cpu.threads‚17 network.recv‚117441 disk.dm-0.in‚ 1982.19140625 disk.nvme0c0n1.in‚ 1852.4375( disk.nvme0c0n1.out‚11310.05712890625( disk.nvme1c1n1.out‚10445.35986328125 disk.nvme1n1.in‚ 129.75390625& disk.nvme1n1.out‚10445.35986328125% proc.memory.availableMB‚ 131.40625 network.sent‚471778 disk.md0.in‚0 disk.dm-0.out‚21136.62890625 disk.dm-1.in‚0$ memory_percent‚99.19795989990234# proc.memory.rssMB‚ 2864.66796875* proc.memory.percent‚17.484545707702637 cpu‚25.78877339139323% disk./.usageGB‚1173.5235557556152 disk.dm-1.out‚ 599.119140625 disk.rbd0.out‚0 disk.nvme0n1.in‚ 1852.4375& disk.nvme0n1.out‚11310.05712890625" disk.nvme1c1n1.in‚ 129.75390625V륓ǂ(:ˆ ÂêòÒ¦âú¤ gpu.0.gpu‚91.0 gpu.0.memory‚17, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚49.0 gpu.0.powerWatts‚217.956' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚72.652 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0e#YÏkÈ‚(:à ÑêòÒ®½Þ¤ network.recv‚122042 disk.dm-1.in‚0& disk.nvme0n1.out‚11393.67724609375" disk.nvme1c1n1.in‚ 129.75390625$ memory_percent‚99.20077323913574 proc.cpu.threads‚17$ disk./.usageGB‚1173.239917755127 disk.dm-1.out‚ 655.02734375 disk.rbd0.out‚0( disk.nvme1c1n1.out‚10518.66845703125 disk.nvme1n1.in‚ 129.75390625 disk.dm-0.in‚ 1982.19140625' proc.memory.availableMB‚ 130.9453125 cpu‚25.78130448832345 network.sent‚529252) disk./.usagePercent‚76.42012165804442 disk.nvme0c0n1.in‚ 1852.4375 disk.nvme0n1.in‚ 1852.4375 disk.md0.in‚0 disk.md0.out‚ 9.55078125" disk.dm-0.out‚21237.0439453125* proc.memory.percent‚17.484593391418457 disk.rbd0.in‚0( disk.nvme0c0n1.out‚11393.67724609375& disk.nvme1n1.out‚10518.66845703125# proc.memory.rssMB‚ 2864.67578125 %•žÉ‚(:“ ÑêòÒɺ¡¥ gpu.0.gpu‚79.0 gpu.0.memory‚13, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚48.0 gpu.0.powerWatts‚226.205' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚75.40166666666667 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0{NðS0Ê+ _wandbruntime‚308  _runtime‚308‹°-ÅË¿ _runtime‚ 308.081098214 _step‚33 ' lr-AdamW/pg1‚0.00013924044332832097 ' lr-AdamW/pg2‚0.00013924044332832097 trainer/global_step‚449 $ _timestamp‚1.784460633692698e+09UiÉÌà trainer/global_step‚449 $ _timestamp‚1.784460633692698e+09 _runtime‚ 308.081098214 _step‚33 ' lr-AdamW/pg1‚0.00013924044332832097 ' lr-AdamW/pg2‚0.00013924044332832097!xî–0Í+ _wandbruntime‚308  _runtime‚308'[ûç έO WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301876224 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105698304 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚185 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚3558821 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚34833006330368 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 369224202795 _step‚34 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚3558821 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12301 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚26110600896000 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825778 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚3558823 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚26492541839360 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 369310989213 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚2384752 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚3414052 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚8074134 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 370329144832 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12372 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12301 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚34791952112077 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚34791848286299 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 370240485888 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚8084277 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚34422624083504 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269299712 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚189 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚34833006330368 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88658944 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10143 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚4525454 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚1029296 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚26110695005696 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 204680192 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚3558823 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚35203246816256 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚3413863 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 370240485888 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚35203352514560 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚34833023369728 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚26492337159168 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 110570496 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 381736263168 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 $ _timestamp‚1.784460633693633e+09 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269299712 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 370329144832 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12372 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2370611712 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786418 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚2384756 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10141 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105698304 HDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚35203246816256 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚4525454 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88658944 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884167 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚34422641122864 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 381846833664 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10143 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301876224 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚4515313 trainer/global_step‚449 _runtime‚ 308.08485128 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200DÁ9ª·'ϱO YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 369224202795 _step‚34 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚3558821 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12301 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚26110600896000 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825778 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚3558823 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚26492541839360 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 369310989213 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚2384752 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚3414052 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚8074134 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 370329144832 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12372 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12301 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚34791952112077 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚34791848286299 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 370240485888 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚8084277 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚34422624083504 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269299712 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚189 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚34833006330368 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88658944 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10143 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚4525454 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚1029296 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚26110695005696 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 204680192 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚3558823 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚35203246816256 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚3413863 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 370240485888 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚35203352514560 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚34833023369728 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚26492337159168 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 110570496 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 381736263168 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 $ _timestamp‚1.784460633693633e+09 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269299712 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 370329144832 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12372 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2370611712 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786418 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚2384756 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10141 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105698304 HDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚35203246816256 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚4525454 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88658944 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884167 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚34422641122864 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 381846833664 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10143 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301876224 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚4515313 trainer/global_step‚449 _runtime‚ 308.08485128 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301876224 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105698304 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚185 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚3558821 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚34833006330368 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048"¦ ]0Ð+ _wandbruntime‚308  _runtime‚308æ%™¶¿&ѹM M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚3421041 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚26166832733696 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 382663367680 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 382546512896 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10157 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 371114867712 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417271 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 _runtime‚ 308.488239257 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚4535288 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884167 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚34908173036032 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12372 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786530 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚34908173036032 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚8101740 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚34496931648256 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105705472 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚3566450 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301876224 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚8091583 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 116854784 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚26549496101376 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2370611712 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12372 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 371114867712 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 370007697395 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269299712 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚34908155996672 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚8101740 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚34908155996672 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 371026201600 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825890 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚4535288 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10155 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚3421219 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚174 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 trainer/global_step‚449 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88666112 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 O6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚2389808 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 370094483925 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301876224 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚4525133 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1031233 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚34867026132181 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚34496914608896 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 210964480 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚34866922306291 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚3566452 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12301 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚1031407 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0¶yµkÒ½M I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10155 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88666112 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚35279182198272 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 _step‚35 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 210964480 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚34866922306291 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚3566452 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12301 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚1031407 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚3421041 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚26166832733696 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 382663367680 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 382546512896 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10157 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 371114867712 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417271 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 _runtime‚ 308.488239257 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚4535288 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884167 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚34908173036032 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12372 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786530 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚34908173036032 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚8101740 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚34496931648256 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105705472 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚3566450 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301876224 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚8091583 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 116854784 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚26549496101376 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2370611712 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12372 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 371114867rí¨4X712 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 370007697395 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269299712 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚34908155996672 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚8101740 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚34908155996672 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 371026201600 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825890 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚4535288 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10155 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚3421219 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚174 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 trainer/global_step‚449 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88666112 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 O6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚2389808 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 370094483925 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301876224 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚4525133 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1031233 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚34867026132181 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚34496914608896 FDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12301 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417271 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚39000185769984 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚39000080071680 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 408549520021 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 _runtime‚ 340.126734746 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚3940271 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚3774898 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚2637556 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825778 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚3940273 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 409590426112 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚38590506684928 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2370611712 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 422487081472 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301876224 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 409679085056 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10141 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10143 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12372 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚28917581405696 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚3940271 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 422368122368 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚3774725 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚1137342 HDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105698304 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚5018306 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚29339855418368 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786418 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12372 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 213068800 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚38545716912485 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88658944 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 409590426112 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚8958579 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚38137150353104 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10143 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚39000080071680 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88658944 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269299712 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚29340068487168 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884167 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚169 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10141 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696P+[4¸'ß²O K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚5008165 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 118959104 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269299712 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 trainer/global_step‚499 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚38590506684928 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 $ _timestamp‚1.784460665737212e+09 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12301 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚38590489645568 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 408462733603 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚8948436 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105698304 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚5018306 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚29339855418368 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786418 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12372 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 213068800 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚38545716912485 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88658944 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 409590426112 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚8958579 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚38137150353104 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10143 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚39000080071680 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88658944 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269299712 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚29340068487168 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884167 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚169 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10141 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12301 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417271 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚39000185769984 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚39000080071680 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 408549520021 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 _runtime‚ 340.126734746 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚3940271 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚3774898 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚2637556 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825778 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚3940273 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 409590426112 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚38590506684928 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2370611712 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 422487081472 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301876224 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 409679085056 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10141 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10143 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12372 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚28917581405696 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚3940271 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 422368122368 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚3774725 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚1137342 HDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚39076121159168 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12301 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88666112 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚39076121159168 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301876224 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚38665639311872 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚8976042 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2370611712 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 410464807936 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417271 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 _step‚39 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269299712 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚29396810260992 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 trainer/global_step‚499 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 410376141824 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚28973719133696 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884167 _runtime‚340.52859174499997 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚3947900 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88666112 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚3947902 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚29397014934016 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786530 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 409246228203 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚5017985 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚38620687106699 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 423185236992 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚38211440878496 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 $ _timestamp‚1.784460666140063e+09 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10157 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚3947900 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚38620790932589 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825890 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚3781953 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚5028140 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚38211457917856 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚8976042 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10155 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301876224 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚162 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 410464807936 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚2642612 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105705472 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12372 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 204673024 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12372 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚8965885 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚38665656351232 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚3782119 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1139345 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 409333014733 O6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 410376141824 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚38665656351232 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚8965885 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚5028140 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚2642608 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486Í€7=Ç&âÁM [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚29397014934016 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786530 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 409246228203 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚5017985 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚38620687106699 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 423185236992 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚38211440878496 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 $ _timestamp‚1.784460666140063e+09 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10157 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚3947900 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚38620790932589 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825890 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚3781953 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚5028140 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚38211457917856 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚8976042 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10155 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301876224 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚162 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 410464807936 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚2642612 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105705472 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12372 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 204673024 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12372 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 B6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚8965885 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚38665656351232 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚3782119 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1139345 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 409333014733 O6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 410376141824 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚38665656351232 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚8965885 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚5028140 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚2642608 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚39076121159168 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12301 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88666112 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚39076121159168 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301876224 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚38665639311872 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚8976042 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2370611712 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 410464807936 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417271 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 _step‚39 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269299712 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚29396810260992 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 trainer/global_step‚499 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 410376141824 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚28973719133696 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884167 _runtime‚340.52859174499997 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚3947900 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88666112 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚3947902'tû¨0ã+ _wandbruntime‚340  _runtime‚340Ê”§äú 0loss/FRt2/assignment_loss‚3.3649947219951586 2loss/SRqqt1/assignment_loss‚1.9244882080671006 2loss/SRqqt2/assignment_loss‚1.5575946136063963 -loss/FBt1/detection_loss‚6.84467328371886 _step‚40 /loss/FRt1/assignment_loss‚4.098960588707306 0loss/FRt1/detection_loss‚0.37176024663797985 0loss/SRqqt1/detection_loss‚3.947065005361197 trainer/global_step‚499 _runtime‚ 340.529080276 1loss/FBt2/assignment_loss‚0.20289936567075195 /loss/FRt2/detection_loss‚1.4517971699672423 1loss/SRqqt2/detection_loss‚0.9812978006606906 %loss/total_loss‚29.24703581677008 epoch‚1 % _timestamp‚1.7844606661409872e+09 1loss/FBt1/assignment_loss‚0.17442991333481123 .loss/FBt2/detection_loss‚4.327074899042583zMråþ /loss/FRt1/assignment_loss‚4.098960588707306 0loss/FRt1/detection_loss‚0.37176024663797985 0loss/SRqqt1/detection_loss‚3.947065005361197 trainer/global_step‚499 _runtime‚ 340.529080276 1loss/FBt2/assignment_loss‚0.20289936567075195 /loss/FRt2/detection_loss‚1.4517971699672423 1loss/SRqqt2/detection_loss‚0.9812978006606906 %loss/total_loss‚29.24703581677008 epoch‚1 % _timestamp‚1.7844606661409872e+09 1loss/FBt1/assignment_loss‚0.17442991333481123 .loss/FBt2/detection_loss‚4.327074899042583 0loss/FRt2/assignment_loss‚3.3649947219951586 2loss/SRqqt1/assignment_loss‚1.9244882080671006 2loss/SRqqt2/assignment_loss‚1.5575946136063963 -loss/FBt1/detection_loss‚6.84467328371886 _step‚40(}ñ\}gæ‚(:Ü þêòÒ½©õ¤ network.sent‚601226 disk.nvme1n1.in‚ 129.75390625 disk.rbd0.out‚0# memory_percent‚99.2069959640503( proc.memory.availableMB‚ 129.92578125# disk.nvme1c1n1.out‚ 10520.890625 disk.md0.in‚0 disk.md0.out‚ 10.44921875 disk.dm-0.in‚ 1982.19140625 disk.dm-1.in‚0( disk.nvme0c0n1.out‚11397.24365234375# proc.memory.rssMB‚ 2864.67578125* proc.memory.percent‚17.484593391418457 network.recv‚133137% disk./.usageGB‚1173.2426147460938" disk.nvme1c1n1.in‚ 129.75390625# disk.dm-0.out‚21240.49853515625 disk.dm-1.out‚655.4033203125 disk.rbd0.in‚0 disk.nvme0c0n1.in‚ 1852.4375 cpu‚25.753634820088404) disk./.usagePercent‚76.42029732917074! disk.nvme1n1.out‚ 10520.890625 disk.nvme0n1.in‚ 1852.4375& disk.nvme0n1.out‚11397.24365234375 proc.cpu.threads‚17àåp^“ç‚(:ˆ þêòÒç×Ü¥ gpu.0.gpu‚84.0 gpu.0.memory‚14, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚49.0 gpu.0.powerWatts‚221.649' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚73.883 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0·ƒÞ=jè‚(:ß ëòÒÖáÔ¤% disk.nvme0n1.out‚11745.1728515625% disk.nvme1n1.out‚10571.8994140625$ memory_percent‚99.20761585235596( proc.memory.availableMB‚ 129.82421875# proc.memory.rssMB‚ 2864.67578125 disk.rbd0.out‚0 network.recv‚137253 disk.md0.in‚0 disk.dm-1.out‚655.5947265625' disk.nvme1c1n1.out‚10571.8994140625* proc.memory.percent‚17.484593391418457( disk./.usagePercent‚76.4511329540308 disk.nvme0c0n1.in‚ 1852.4375" disk.nvme1c1n1.in‚ 129.75390625 cpu‚25.74616251524045 proc.cpu.threads‚17' disk.nvme0c0n1.out‚11745.1728515625 network.sent‚663879% disk./.usageGB‚1173.7160186767578 disk.md0.out‚ 10.7734375" disk.dm-0.out‚21638.5400390625 disk.dm-1.in‚0 disk.nvme0n1.in‚ 1852.4375 disk.nvme1n1.in‚ 129.75390625 disk.dm-0.in‚ 1982.19140625 disk.rbd0.in‚0Ɇ¼­Ÿé‚(:” ëòÒÑø„¥ gpu.0.gpu‚91.0 gpu.0.memory‚21, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚48.0 gpu.0.powerWatts‚144.545' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚48.181666666666665 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0oÞ~"0ê+ _wandbruntime‚372  _runtime‚372ÙÒϵÃë½ & lr-AdamW/pg2‚0.0001392227364492536 trainer/global_step‚549 $ _timestamp‚1.784460698118788e+09 _runtime‚ 372.507673794 _step‚41 & lr-AdamW/pg1‚0.0001392227364492536¾£|ÇÇìÁ & lr-AdamW/pg1‚0.0001392227364492536 & lr-AdamW/pg2‚0.0001392227364492536 trainer/global_step‚549 $ _timestamp‚1.784460698118788e+09 _runtime‚ 372.507673794 _step‚41)è`ç0í+ _wandbruntime‚372  _runtime‚372ðk”Gµ'î¯O KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10141 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚2890356 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105698304 HDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269299712 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚202 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417271 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 _runtime‚ 372.507948645 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚42797019025408 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10143 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 449029025280 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚4321723 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105698304 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚4321723 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚41851676622704 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269299712 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚42347972960768 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚4321721 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚32187550488576 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚31724373696000 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 448940366336 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10141 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2370611712 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚42299377887115 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚4321721 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 223554560 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786418 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 447701264411 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884167 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚42347990000128 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88658944 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚206 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚32187326934016 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚31724467805696 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 % _timestamp‚1.7844606981194513e+09 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12372 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 129444864 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 HDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚42347972960768 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 449029025280 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10143 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚5501017 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚42347990000128 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 462953238016 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚41851693662064 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 448940366336 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚4135482 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 447788050829 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚5511158 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚9822738 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12301 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚1245126 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825778 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301876224 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12301 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚42797019025408 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚42796913327104 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚5511158 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚5501017 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 463082682880 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚42299481712893 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚9832881 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269299712 QIDeviceStatsMonitor.on_train_batch_startŽ¨˜èð!/inactive_split.small_pool.current‚202 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417271 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 _runtime‚ 372.507948645 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚42797019025408 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10143 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 449029025280 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚4321723 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105698304 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚4321723 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚41851676622704 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269299712 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚42347972960768 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚4321721 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚32187550488576 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚31724373696000 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 448940366336 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10141 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2370611712 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚42299377887115 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚4321721 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 223554560 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786418 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 447701264411 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884167 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚42347990000128 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88658944 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚206 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚32187326934016 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚31724467805696 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 % _timestamp‚1.7844606981194513e+09 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12372 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 129444864 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 HDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚42347972960768 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 449029025280 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10143 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚5501017 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚42347990000128 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 462953238016 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚41851693662064 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 448940366336 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚4135482 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 447788050829 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚5511158 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 D8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚9822738 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12301 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚1245126 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825778 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301876224 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12301 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚42797019025408 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚42796913327104 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚5511158 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚5501017 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 463082682880 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚42299481712893 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 L@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚9832881 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301876224 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚1247254 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚32244283185664 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚42374451907107 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚41925967148096 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚9850344 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884167 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786530 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88666112 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚42872954414592 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚31780605533696 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105705472 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105705472 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 448571545541 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 % _timestamp‚1.7844606985212896e+09 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _step‚43 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚5510837 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚166 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚42423139666432 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 463771761664 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10155 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10157 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 449814748160 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚9850344 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚2895412 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 215158784 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825890 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301876224 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12372 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚4142666 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚31780511424000 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚41925984187456 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269299712 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 121049088 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚42374555732997 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ODeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚4142500 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2370611712 trainer/global_step‚549 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 449726082048 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚9840187 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚4329352 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10155 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417271 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 449814748160 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12372 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚42872954414592 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚162 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269299712 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚166 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚42423139666432 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 463771761664 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10155 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10157 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 449814748160 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 F:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚9850344 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚2895412 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 215158784 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825890 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301876224 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12372 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚4142666 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚31780511424000 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚41925984187456 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269299712 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 121049088 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚42374555732997 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ODeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚4142500 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2370611712 trainer/global_step‚549 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 449726082048 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 F:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚9840187 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚4329352 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10155 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417271 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 449814748160 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12372 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚42872954414592 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚162 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269299712 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301876224 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚1247254 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚32244283185664 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚42374451907107 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚41925967148096 J>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚9850344 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884167 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786530 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88666112 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚42872954414592 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚31780605533696 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105705472 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105705472 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 448571545541 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 % _timestamp‚1.7844606985212896e+09 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _step‚43 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚5510837 FBkù‚(:‘ «ëòÒ¹—¦¥ gpu.0.gpu‚91.0 gpu.0.memory‚19, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚50.0 gpu.0.powerWatts‚116.9' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚38.96666666666667 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoÔ–G†1ryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ÕyIµ0ú+ _wandbruntime‚404  _runtime‚404w#QÆûÀ % _timestamp‚1.7844607296965675e+09 _runtime‚ 404.085490359 _step‚45 ' lr-AdamW/pg1‚0.00013920717709785211 ' lr-AdamW/pg2‚0.00013920717709785211 trainer/global_step‚599ZÎÆÊüÄ trainer/global_step‚599 % _timestamp‚1.7844607296965675e+09 _runtime‚ 404.085490359 _step‚45 ' lr-AdamW/pg1‚0.00013920717709785211 ' lr-AdamW/pg2‚0.00013920717709785211-¯OWp0ý+ _wandbruntime‚404  _runtime‚404:?“¹'þ³O JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105698304 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 trainer/global_step‚599 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚4703173 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10141 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚4495142 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚46593746582528 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚5993869 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚3143152 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚46053246513301 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚46105456275968 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚3143156 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚35035002267136 _runtime‚ 404.085551049 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚4703171 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚10707183 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12372 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚1352172 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 127347712 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 % _timestamp‚1.7844607296972132e+09 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚46593746582528 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 488290306560 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 486939795219 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417271 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚4703171 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚182 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 503520713728 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚46053142687523 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884167 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10143 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚10697040 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚46105473315328 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825778 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12301 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88658944 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105698304 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚46593852280832 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚46105473315328 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚1351990 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 221457408 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚4703173 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚45566202892304 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786418 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301876224 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10141 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚4495328 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚34531354205696 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10143 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 trainer/global_step‚599 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚4703173 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10141 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚4495142 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚46593746582528 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚5993869 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚3143152 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚46053246513301 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚46105456275968 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚3143156 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚35035002267136 _runtime‚ 404.085551049 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚4703171 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚10707183 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12372 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚1352172 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 127347712 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 % _timestamp‚1.7844607296972132e+09 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚46593746582528 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 488290306560 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 486939795219 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417271 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚4703171 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚182 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 503520713728 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚46053142687523 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884167 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10143 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚10697040 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚46105473315328 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825778 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12301 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88658944 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105698304 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚46593852280832 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚46105473315328 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚1351990 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 221457408 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚4703173 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚45566202892304 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786418 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301876224 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10141 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚4495328 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚34531354205696 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10143 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812.kË3ý0€+ _wandbruntime‚404  _runtime‚404ƒýò‹Ã&½M MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105705472 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 133632000 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚6003689 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12372 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚34587491933696 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚46128216707515 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚4710800 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚46180605942272 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2370611712 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 504463825408 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1354030 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 % _timestamp‚1.7844607301064801e+09 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚10714489 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 _runtime‚ 404.495002434 ODeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚10724646 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105705472 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269299712 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12372 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚46128320533405 _step‚47 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚46669787670016 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚46180622981632 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 489164688384 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚35091728017408 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 489076022272 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚10724646 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10155 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 227741696 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 504330193408 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301876224 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚46669787670016 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚10714489 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10157 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚4710800 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚45640493417696 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 487723289819 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚6013844 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚6003689 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚4710802 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417271 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚4502428 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚190 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825890 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚4502238 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚6013844 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚1354216 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10155 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884167 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚45640510457056 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 487810076349 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚4710802 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚34587397824000 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 489164688384 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88666112 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚4502238 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚6013844 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚1354216 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10155 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884167 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚45640510457056 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 [JDeviceStatsMo¹N맗nitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 487810076349 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚4710802 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚34587397824000 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 489164688384 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88666112 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 % _timestamp‚1.7844607301064801e+09 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚10714489 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 _runtime‚ 404.495002434 ODeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚10724646 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105705472 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269299712 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12372 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚46128320533405 _step‚47 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚46669787670016 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚46180622981632 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 489164688384 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚35091728017408 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 489076022272 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚10724646 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10155 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 227741696 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 504330193408 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301876224 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚46669787670016 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚10714489 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10157 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚4710800 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚45640493417696 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 487723289819 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚6013844 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚6003689 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚4710802/ÄÖ¸F0ƒ+ _wandbruntime‚404  _runtime‚404:#×Áþ„ø /loss/FRt1/detection_loss‚0.5084006520201096 .loss/FRt2/detection_loss‚1.366768952038949 0loss/SRqqt1/detection_loss‚2.883624026581014 /loss/FBt1/detection_loss‚11.275563876563913 % _timestamp‚1.7844607301074178e+09 _step‚48 0loss/FRt2/assignment_loss‚2.9955338782611047 1loss/SRqqt1/assignment_loss‚1.877638144267572 1loss/SRqqt2/assignment_loss‚1.461899543233197 .loss/FBt2/detection_loss‚3.960528381628863 trainer/global_step‚599 0loss/FRt1/assignment_loss‚4.0450924932956696 0loss/FBt1/assignment_loss‚0.1184734958884536 1loss/FBt2/assignment_loss‚0.15529619158041896 1loss/SRqqt2/detection_loss‚0.9276290466915934 %loss/total_loss‚31.57644868205086 epoch‚1 _runtime‚ 404.495541125sZ·õ…ü 0loss/FRt1/assignment_loss‚4.0450924932956696 0loss/FBt1/assignment_loss‚0.1184734958884536 1loss/FBt2/assignment_loss‚0.15529619158041896 1loss/SRqqt2/detection_loss‚0.9276290466915934 %loss/total_loss‚31.57644868205086 epoch‚1 _runtime‚ 404.495541125 /loss/FRt1/detection_loss‚0.5084006520201096 .loss/FRt2/detection_loss‚1.366768952038949 0loss/SRqqt1/detection_loss‚2.883624026581014 /loss/FBt1/detection_loss‚11.275563876563913 % _timestamp‚1.7844607301074178e+09 _step‚48 0loss/FRt2/assignment_loss‚2.9955338782611047 1loss/SRqqt1/assignment_loss‚1.877638144267572 1loss/SRqqt2/assignment_loss‚1.461899543233197 .loss/FBt2/detection_loss‚3.960528381628863 trainer/global_step‚5990C¤fo†‚(:ä ºëòÒ¤Ÿ¥ network.sent‚748606) disk./.usagePercent‚76.27415609678098# disk./.usageGB‚1170.99898147583& disk.nvme0n1.out‚11855.73779296875 disk.md0.in‚0 disk.dm-1.in‚0! disk.dm-1.out‚655.95458984375 disk.rbd0.out‚0 disk.nvme0n1.in‚ 1852.4453125 disk.nvme1c1n1.in‚ 129.765625& disk.nvme1n1.out‚10576.85302734375 disk.md0.out‚ 11.91796875" disk.nvme0c0n1.in‚ 1852.4453125# proc.memory.rssMB‚ 2864.67578125 cpu‚25.73216120426516 proc.cpu.threads‚17 network.recv‚148857 disk.rbd0.in‚0( disk.nvme0c0n1.out‚11855.73779296875$ memory_percent‚99.21064376831055* proc.memory.percent‚17.484593391418457( disk.nvme1c1n1.out‚10576.85302734375 disk.nvme1n1.in‚ 129.765625 disk.dm-0.in‚ 1982.2109375# disk.dm-0.out‚21751.21435546875& proc.memory.availableMB‚ 129.328125žÞÃŇ‚(:’ ºëòÒüԥ gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚49.0 gpu.0.powerWatts‚114.936' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚38.312000000000005 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ô’Ø¥mˆ‚(:â ÉëòÒÚºÛ¤# memory_percent‚99.2110013961792* proc.memory.percent‚17.484593391418457) disk./.usagePercent‚76.27416926590361' disk.nvme0c0n1.out‚11856.5791015625# disk.dm-0.out‚21751.83544921875( proc.memory.availableMB‚ 129.26953125 network.recv‚153543 disk.dm-1.in‚0 disk.rbd0.in‚0 disk.md0.in‚0 cpu‚25.725923311505124 proc.cpu.threads‚17% disk./.usageGB‚1170.9991836547852 disk.dm-1.out‚656.0830078125 disk.nvme0n1.in‚ 1852.4453125( disk.nvme1c1n1.out‚10577.41259765625& disk.nvme1n1.out‚10577.41259765625 disk.md0.out‚12.21875 disk.dm-0.in‚ 1982.2109375# proc.memory.rssMB‚ 2864.67578125 network.sent‚800304 disk.rbd0.out‚0" disk.nvme0c0n1.in‚ 1852.4453125% disk.nvme0n1.out‚11856.5791015625 disk.nvme1c1n1.in‚ 129.765625 disk.nvme1n1.in‚ 129.765625ûDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12301 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚6486721 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚49280746201264 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786418 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚5084623 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚5084621 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12301 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚3395952 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚49862956630528 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚37338146496000 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚49807011313709 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚50390579837952 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12372 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10143 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10141 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚49862939591168 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚5084623 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10141 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2370611712 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚6486721 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 527640246784 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚11581485 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚37882234799104 trainer/global_step‚649 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269299712 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚49806907487931 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚49280729161904 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417271 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 527728905728 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚1458900 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88658944 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12372 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚4854852 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚166 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚11571342 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚49862956630528 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 221457408 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚37338240605696 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 544215650816 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚170 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚50390685536256 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 527728905728 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10143 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 _runtime‚ 435.955949106 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88658944 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 127347712 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚50390579837952 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301876224 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105698304 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚6496862 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 527640246784 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚1459066 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚3395956 % _timestamp‚1.7844607615665464e+09 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 526265112445 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚49862939591168 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚5084621 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚6496862 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚4855022 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 Bº˜!‘·O E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 526178326027 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105698304 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269299712 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 544088303104 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12301 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚6486721 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚49280746201264 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786418 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚5084623 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚5084621 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12301 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚3395952 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚49862956630528 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚37338146496000 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚49807011313709 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚50390579837952 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12372 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10143 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10141 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚49862939591168 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚5084623 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10141 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2370611712 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚6486721 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 527640246784 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚11581485 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚37882234799104 trainer/global_step‚649 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269299712 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚49806907487931 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚49280729161904 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417271 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 527728905728 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚1458900 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88658944 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12372 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚4854852 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚166 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚11571342 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚49862956630528 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 221457408 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚37338240605696 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 544215650816 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚170 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚50390685536256 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 527728905728 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10143 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 _runtime‚ 435.955949106 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88658944 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 127347712 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚50390579837952 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 YKDeviceStatsMonitor.on_train_batch_startûÿά/reserved_bytes.small_pool.allocated‚ 272629760 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301876224 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105698304 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚6496862 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 527640246784 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚1459066 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚3395956 % _timestamp‚1.7844607615665464e+09 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 526265112445 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚49862939591168 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚5084621 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚6496862 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚4855022 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚6506696 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚49938106296832 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚37394378333696 _runtime‚ 436.360138358 _step‚51 ODeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417271 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚167 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 527048607157 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚5092250 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105705472 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10155 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10157 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12372 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚5092250 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚3401012 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12301 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚11588791 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 528425962496 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚6496541 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 % _timestamp‚1.7844607619709682e+09 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚50466620925440 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚6506696 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10155 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105705472 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88666112 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 528514628608 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786530 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚11598948 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚50466620925440 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269299712 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚11588791 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚5092252 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚49938089257472 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301876224 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825890 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚50466515219968 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884167 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚11598948 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12372 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 123146240 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚49938106296832 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 528514628608 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚4861980 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486*†V´Ç&”ÁM trainer/global_step‚649 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚49938089257472 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚6496541 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚4862151 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚6506696 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚49938106296832 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚37394378333696 _runtime‚ 436.360138358 _step‚51 ODeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417271 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚167 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 527048607157 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚5092250 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105705472 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10155 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10157 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12372 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚5092250 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚3401012 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12301 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚11588791 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 528425962496 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚6496541 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 % _timestamp‚1.7844607619709682e+09 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚50466620925440 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚6506696 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10155 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105705472 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88666112 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 528514628608 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786530 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚11598948 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚50466620925440 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269299712 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚11588791 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚5092252 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚49938089257472 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301876224 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825890 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚50466515219968 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884167 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚11598948 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12372 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 123146240 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚49938106296832 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 528514628608 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚4861980 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚37939403227648 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 544901747712 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1460972 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚49882085333813 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚49881981507923 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 528425962496 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269299712 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚379391859717123;ÄÕ0•+ _wandbruntime‚436  _runtime‚436õƒÆð–ú 2loss/SRqqt2/assignment_loss‚1.3888876860389712 1loss/SRqqt1/detection_loss‚3.0506650754595737 .loss/FRt1/assignment_loss‚3.98885912025297 /loss/FRt2/detection_loss‚1.3398572414577758 1loss/SRqqt2/detection_loss‚0.9928124061435777 .loss/FBt1/detection_loss‚6.785063455743187 .loss/FBt2/detection_loss‚4.108643382409109 1loss/FBt1/assignment_loss‚0.13225177289256765 trainer/global_step‚649 $ _timestamp‚1.784460761971972e+09 _runtime‚ 436.360466999 &loss/total_loss‚27.145687183417053 epoch‚1 2loss/SRqqt1/assignment_loss‚1.7396032811512594 1loss/FBt2/assignment_loss‚0.24360918678298948 /loss/FRt1/detection_loss‚0.4748576563521842 _step‚52 0loss/FRt2/assignment_loss‚2.9005769187328863Õ ÞÄ—þ 1loss/FBt1/assignment_loss‚0.13225177289256765 trainer/global_step‚649 $ _timestamp‚1.784460761971972e+09 _runtime‚ 436.360466999 &loss/total_loss‚27.145687183417053 epoch‚1 2loss/SRqqt1/assignment_loss‚1.7396032811512594 1loss/FBt2/assignment_loss‚0.24360918678298948 /loss/FRt1/detection_loss‚0.4748576563521842 _step‚52 0loss/FRt2/assignment_loss‚2.9005769187328863 2loss/SRqqt2/assignment_loss‚1.3888876860389712 1loss/SRqqt1/detection_loss‚3.0506650754595737 .loss/FRt1/assignment_loss‚3.98885912025297 /loss/FRt2/detection_loss‚1.3398572414577758 1loss/SRqqt2/detection_loss‚0.9928124061435777 .loss/FBt1/detection_loss‚6.785063455743187 .loss/FBt2/detection_loss‚4.1086433824091094kN—f˜‚(:Û çëòÒ½óƤ% disk.nvme1n1.out‚10579.8486328125 disk.dm-1.out‚ 656.255859375 disk.rbd0.out‚0 disk.nvme0n1.in‚ 1852.4453125* proc.memory.percent‚17.484593391418457 cpu‚25.71725143491254 network.sent‚808876% disk./.usageGB‚1173.7608680725098 disk.nvme1n1.in‚ 129.765625 disk.md0.out‚ 12.9609375 disk.rbd0.in‚0& disk.nvme0c0n1.out‚13777.322265625# proc.memory.rssMB‚ 2864.67578125 disk.md0.in‚0 disk.dm-0.in‚ 1982.2109375" disk.dm-0.out‚23673.2451171875$ disk.nvme0n1.out‚13777.322265625 disk.nvme1c1n1.in‚ 129.765625% proc.memory.availableMB‚ 3.6640625) disk./.usagePercent‚76.45405426298713 disk.dm-1.in‚0" disk.nvme0c0n1.in‚ 1852.4453125' disk.nvme1c1n1.out‚10579.8486328125$ memory_percent‚99.97763633728027 proc.cpu.threads‚17 network.recv‚161399²‘“ž™‚(:“ çëòÒã©¥ gpu.0.gpu‚84.0 gpu.0.memory‚20, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚50.0 gpu.0.powerWatts‚122.855' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚40.95166666666667 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0(÷kš‚(:à öëòÒ¿Æø¤ disk.md0.in‚0 disk.dm-0.in‚ 1982.2109375# disk.dm-0.out‚24546.22509765625 disk.dm-1.in‚0 disk.nvme0n1.in‚ 1852.4453125 disk.dm-1.out‚ 656.291015625" disk.nvme0c0n1.in‚ 1852.4453125$ memory_percent‚99.97777938842773 cpu‚25.71196403182273 proc.cpu.threads‚17) disk./.usagePercent‚76.45104474575662% disk./.usageGB‚1173.7146644592285 disk.md0.out‚ 13.20703125 disk.rbd0.in‚0 disk.rbd0.out‚0( disk.nvme0c0n1.out‚14522.64404296875& disk.nvme0n1.out‚14522.64404296875$ proc.memory.availableMB‚3.640625 network.sent‚873556 disk.nvme1c1n1.in‚ 129.765625' disk.nvme1c1n1.out‚10708.0771484375 disk.nvme1n1.in‚ 129.765625# proc.memory.rssMB‚ 2864.67578125* proc.memory.percent‚17.484593391418457 network.recv‚166216% disk.nvme1n1.out‚10708.0771484375LsŸ,›‚(:’ öëòÒÔ¾«¥ gpu.0.gpu‚32.0 gpu.0.memory‚8, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚49.0 gpu.0.powerWatts‚115.109' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚38.36966666666666 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0x¯d0œ+ _wandbruntime‚468  _runtime‚468H¬KÆÀ ' lr-AdamW/pg1‚0.00013916265156798966 ' lr-AdamW/pg2‚0.00013916265156798966 trainer/global_step‚699 % _timestamp‚1.7844607938026304e+09 _runtime‚ 468.191001183 _step‚53,/ÊžÄ % _timestamp‚1.7844607938026304e+09 _runtime‚ 468.191001183 _step‚53 ' lr-AdamW/pg1‚0.00013916265156798966 ' lr-AdamW/pg2‚0.00013916265156798966 trainer/global_step‚6995±e$ß0Ÿ+ _wandbruntime‚468  _runtime‚468†_ªú ³O QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105698304 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12372 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚40145032896000 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 565503643253 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚40729877376512 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10143 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚5466073 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12301 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚5466071 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚3648752 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301876224 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚5214063 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚53620422906368 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88658944 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10141 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚53620439945728 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚6989714 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚52995272470864 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786418 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88658944 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚3648756 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚54187413093376 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚54187518791680 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269299712 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚40145127005696 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 129444864 _runtime‚ 468.191874985 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884167 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417271 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10143 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2370611712 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚6989714 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚54187413093376 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12301 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 584750370816 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 584620925952 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚53560672288339 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105698304 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12372 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301876224 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 566990187008 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 567078845952 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 566990187008 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚179 trainer/global_step‚699)ì½'¡·O _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚53620439945728 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚1565307 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚1565132 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚40729653821952 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚6979573 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105698304 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12372 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301876224 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 566990187008 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 567078845952 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 566990187008 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚179 trainer/global_step‚699 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105698304 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12372 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚40145032896000 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 565503643253 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚40729877376512 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10143 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚5466073 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12301 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚5466071 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚3648752 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301876224 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚5214063 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚53620422906368 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88658944 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10141 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚53620439945728 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚6989714 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚52995272470864 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786418 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88658944 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚3648756 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚54187413093376 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚54187518791680 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269299712 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚40145127005696 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 129444864 _runtime‚ 468.191874985 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884167 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417271 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10143 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2370611712 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚6989714 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚54187413093376 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12301 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 584750370816 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 584620925952 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚53560672288339 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 IDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105705472 O6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚5221230 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚12463093 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚3653812 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚40201264733696 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 219353088 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚53069562996256 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 566200351435 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105705472 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚6999548 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 trainer/global_step‚699 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10157 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269299712 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884167 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786530 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2370611712 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚171 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 567864568832 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚12473250 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417271 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88666112 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚12463093 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚5221059 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚53069545956896 _step‚55 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚54263454180864 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚40201170624000 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚53635850134221 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚6989393 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301876224 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚53695572572672 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12372 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825890 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10155 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚6989393 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚40786822864896 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88666112 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 585558131200 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚6999548 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚54263348475392 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12301 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚3653808 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130²Äcn"¤ÁM UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 585432887808 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚53695572572672 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10155 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1567251 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10155 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚6989393 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚40786822864896 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88666112 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 585558131200 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚6999548 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚54263348475392 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12301 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚3653808 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105705472 O6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚5221230 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚12463093 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚3653812 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚40201264733696 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 219353088 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚53069562996256 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 566200351435 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105705472 QEDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚6999548 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 trainer/global_step‚699 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10157 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269299712 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884167 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786530 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2370611712 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚171 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 567864568832 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚12473250 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417271 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88666112 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚12463093 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚5221059 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚53069545956896 _step‚55 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚54263454180864 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚40201170624000 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚53635850134221 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚6989393 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301876224 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚53695572572672 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12372 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825890 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 FDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10143 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚57314540914525 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚5847521 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 606428786176 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105698304 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚56709781701104 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10141 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚57984246348800 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269299712 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786418 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚57984246348800 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10141 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10143 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 625131100672 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚56709798740464 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 606340127232 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚3901552 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88658944 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2370611712 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884167 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 trainer/global_step‚749 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚7482566 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301876224 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚43577050396672 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚13319946 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚5847523 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12301 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚1671468 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825778 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417271 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 604742174061 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚5573020 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚3901556 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚57377906221568 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚57377906221568 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 % _timestamp‚1.7844608259621725e+09 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88658944 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 606428786176 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚5847521 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12301 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚57984352047104 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _step‚58 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269299712 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚57314437088747 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚57377923260928 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚7472425 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚1671640 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚172 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚42952013405696 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚5573196 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 _runtime‚ 500.350413597¢£§˜½'±·O YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 _runtime‚ 500.350413597 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚13330089 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚7482566 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105698304 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301876224 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10143 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚57314540914525 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚5847521 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 606428786176 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105698304 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚56709781701104 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10141 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚57984246348800 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269299712 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786418 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚57984246348800 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10141 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10143 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 625131100672 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚56709798740464 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 606340127232 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚3901552 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88658944 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2370611712 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884167 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 trainer/global_step‚749 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚7482566 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301876224 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚43577050396672 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚13319946 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚5847523 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12301 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚1671468 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825778 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417271 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 604742174061 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚5573020 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚3901556 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚57377906221568 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚57377906221568 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 % _timestamp‚1.7844608259621725e+09 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88658944 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 606428786176 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚5847521 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12301 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚57984352047104 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _step‚58 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269299712 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚57314437088747 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚57377923260928 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚7472425 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚1671640 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚172 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚42952013405696 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚5573196 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488:̲<0²+ _wandbruntime‚500  _runtime‚5002Ÿr¥e³½M TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 607125842944 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2370611712 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚43008151133696 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 605438882243 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚13337395 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚5855152 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚57453072927232 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10157 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚57453055887872 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884167 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚58060181730816 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚5855152 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚43008057024000 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10155 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚1673694 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10155 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚13347552 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 626070951936 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 % _timestamp‚1.7844608263665745e+09 _runtime‚ 500.755125884 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚13347552 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚58060287436288 ODeviceStatsMonitor.on_train_batch_end/inacŽ6Ér^ tive_split.all.freed‚5580116 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 trainer/global_step‚749 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105705472 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825890 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417271 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚7492400 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚7482245 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269299712 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚186 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚56784089265856 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 605525668773 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚3906608 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚57389511108739 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301876224 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1673508 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105705472 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚57453072927232 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 607214509056 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚190 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 225644544 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚43633996441088 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 131534848 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 _step‚59 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12372 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 607214509056 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88666112£úJ Ç&´ÁM G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚13337395 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12301 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884167 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚58060181730816 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚5855152 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚43008057024000 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10155 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚1673694 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10155 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚13347552 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 626070951936 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 % _timestamp‚1.7844608263665745e+09 _runtime‚ 500.755125884 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚13347552 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚58060287436288 ODeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚5580116 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 trainer/global_step‚749 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105705472 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825890 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417271 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚7492400 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚7482245 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269299712 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚186 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚56784089265856 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 605525668773 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚3906608 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚57389511108739 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301876224 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1673508 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105705472 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚57453072927232 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 607214509056 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚190 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 225644544 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚43633996441088 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 131534848 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 _step‚59 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12372 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 607214509056 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88666112 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 607125842944 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2370611712 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚43008151133696 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 605438882243 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚13337395 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚5855152 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚57453072927232 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10157 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚57453055887872 FDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10143 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚6228973 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚45758899805696 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚4154352 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 _step‚62 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105698304 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12301 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚61068201889155 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10143 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚61135389536768 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚14204391 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12301 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88658944 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301876224 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚60424307970704 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚5932652 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 645778726400 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚61781079604224 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 665753513472 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0U)XŒÁ¶O OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚6228973 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚45758899805696 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚4154352 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 _step‚62 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105698304 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12301 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚61068201889155 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10143 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚61135389536768 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚14204391 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12301 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88658944 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301876224 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚60424307970704 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚5932652 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 645778726400 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚61781079604224 JBDeviceStatsMonitor.on_train_batch_start/segœïï0ment.large_pool.current‚682 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 665753513472 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 _runtime‚ 533.734845562 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚164 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚60424325010064 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 643893918451 $ _timestamp‚1.784460859343306e+09 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚14194248 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚61781185302528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88658944 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825778 trainer/global_step‚799 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10141 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚61781079604224 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269299712 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚7975418 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚1778300 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚45758805696000 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚7965277 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12372 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚160 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 123153408 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚7975418 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚46424653319168 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786418 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12372 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚4154356 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 645690067456 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚1778460 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚46424436056064 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 665630360064 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚7965277 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚6228973 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚61135389536768 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 645690067456 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 643980704869 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚5932816 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269299712 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚61135406576128 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚6228971 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10143 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200>UŽË±0Â+ _wandbruntime‚534  _runtime‚534k=™Ã&ýM VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚61857120691712 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 646475783168 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚6236600 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 646564449280 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884167 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 644677413051 trainer/global_step‚799 O6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 666564001792 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417271 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚5939869 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚169 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88666112 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269299712 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 221450240 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚45814943424000 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚173 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚4159408 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1780288 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚14221854 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚5939696 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚1780457 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚61857014986240 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301876224 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚14211697 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚61210539203072 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 646564449280 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚45815037533696 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 _runtime‚ 534.143117368 % _timestamp‚1.7844608597545898e+09 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301876224 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚61210556242432 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10155 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚4159412 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚61143379735037 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚60498598496096 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2370611712 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚60498615535456 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚61210539203072 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105705472 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12372 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚7975097 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88666112 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚14221854 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚7975097 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12372 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚61857120691712 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚61210556242432»,PäÇ&ÄÁM MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚7985252 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 666436661248 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚61857120691712 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚61210556242432 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚61857120691712 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 646475783168 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚6236600 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 646564449280 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884167 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 644677413051 trainer/global_step‚799 O6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 666564001792 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417271 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚5939869 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚169 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88666112 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269299712 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 221450240 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚45814943424000 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚173 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚4159408 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1780288 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚14221854 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚5939696 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚1780457 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚61857014986240 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301876224 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚14211697 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚61210539203072 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 646564449280 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚45815037533696 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 _runtime‚ 534.143117368 % _timestamp‚1.7844608597545898e+09 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301876224 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚61210556242432 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10155 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚4159412 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚61143379735037 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚60498598496096 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2370611712 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚60498615535456 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚61210539203072 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105705472 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12372 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚7975097 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88666112 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚14221854 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚7975097 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12372 F»Ø0Ê+ _wandbruntime‚547  _runtime‚547/¿:lˇ| =&Purity/*FRt0SRqqt2FBt/event_proportion‚0.0103004090487957 9"Purity/0FRt*SRqqt1FBt/event_purity‚0.7674689292907715 8 Purity/1FRt2SRqqt0FBt/FRt_purity‚0.39152753353118896 ,jet/accuracy_3_of_3‚0.019393548369407654 8 Purity/*FRt1SRqqt1FBt/FRt_purity‚0.29602086544036865 7 Purity/0FRt*SRqqt1FBt/FBt_purity‚0.9865090250968933 9"Purity/0FRt0SRqqt1FBt/event_purity‚0.9876819849014282 @&Purity/1FRt0SRqqt2FBt/event_proportion‚0.0024879092816263437 :"Purity/*FRt1SRqqt2FBt/SRqqt_purity‚0.37037038803100586 =&Purity/1FRt*SRqqt0FBt/event_proportion‚0.6105840802192688 =&Purity/*FRt0SRqqt1FBt/event_proportion‚0.0418526791036129 7 Purity/*FRt1SRqqt*FBt/FBt_purity‚0.9683840870857239 :"Purity/1FRt2SRqqt0FBt/SRqqt_purity‚0.15531933307647705 ?&Purity/2FRt2SRqqt*FBt/event_proportion‚0.006417410913854837 ("Purity/2FRt1SRqqt2FBt/event_purity‚0 _runtime‚ 547.617006062 =&Purity/*FRt1SRqqt1FBt/event_proportion‚0.0188569575548172 & Purity/*FRt2SRqqt1FBt/FBt_purity‚1 ("Purity/1FRt2SRqqt1FBt/event_purity‚0 @&Purity/2FRt1SRqqt2FBt/event_proportion‚2.325148852833081e-05 *jet/accuracy_1_of_3‚0.6444879174232483 7 Purity/*FRt*SRqqt1FBt/FBt_purity‚0.9882218241691589 =&Purity/1FRt0SRqqt0FBt/event_proportion‚0.4408714771270752 7 Purity/1FRt1SRqqt*FBt/FRt_purity‚0.4671282172203064 ;"Purity/1FRt1SRqqt0FBt/event_purity‚0.004130990710109472 >&Purity/2FRt0SRqqt0FBt/event_proportion‚0.01990327425301075 =&Purity/*FRt*SRqqt1FBt/event_proportion‚0.0617094486951828 =&Purity/0FRt*SRqqt1FBt/event_proportion‚0.0436662957072258 :"Purity/1FRt0SRqqt1FBt/event_purity‚0.20962484180927277 <&Purity/*FRt*SRqqt0FBt/event_proportion‚0.881045401096344 9 Purity/*FRt0SRqqt*FBt/FRt_purity‚0.048534296452999115 :"Purity/0FRt1SRqqt1FBt/SRqqt_purity‚0.38826343417167664 -"Purity/0FRt2SRqqt1FBt/SRqqt_purity‚0.1875 @&Purity/2FRt0SRqqt1FBt/event_proportion‚0.0007672990905120969 6 Purity/*FRt0SRqqt2FBt/FBt_purity‚0.929855465888977 :"Purity/*FRt2SRqqt0FBt/SRqqt_purity‚0.15093088150024414 & Purity/0FRt2SRqqt1FBt/FBt_purity‚1 7 Purity/1FRt2SRqqt*FBt/FRt_purity‚0.3886665999889374 & Purity/2FRt1SRqqt1FBt/FBt_purity‚1 & Purity/2FRt0SRqqt1FBt/FBt_purity‚1 9"Purity/*FRt*SRqqt2FBt/event_purity‚0.6427225470542908 :"Purity/*FRt1SRqqt1FBt/event_purity‚0.15715990960597992 :"Purity/0FRt*SRqqt0FBt/event_purity‚0.12270773947238922 7 Purity/1FRt1SRqqt2FBt/FBt_purity‚0.7777777910232544 9"Purity/*FRt0SRqqt2FBt/event_purity‚0.6868665814399719 8&Purity/*FRt2SRqqt0FBt/event_proportion‚ 0.05322265625 7 Purity/0FRt0SRqqt2FBt/FBt_purity‚0.9205908179283142 7 Purity/2FRt2SRqqt0FBt/FRt_purity‚0.2159012109041214 jet/accuracy_2_of_5‚0 ;"Purity/*FRt*SRqqt0FBt/event_purity‚0.054136067628860474 8 Purity/*FRt1SRqqt0FBt/FRt_purity‚0.44761916995048523 <"Purity/2FRt0SRqqt*FBt/event_purity‚0.0036614774726331234 ("Purity/2FRt1SRqqt0FBt/event_purity‚0 7 Purity/1FRt1SRqqt2FBt/FRt_purity‚0.5555555820465088 particle/accuracy_1_of_1‚1 _step‚65 ;"Purity/2FRt*SRqqt1FBt/event_purity‚0.009999999776482582 particle/accuracy_2_of_3‚1 ("Purity/0FRt1SRqqt2FBt/event_purity‚0 8 Purity/*FRt0SRqqt2FBt/FRt_purity‚0.45891907811164856 >&Purity/0FRt2SRqqt*FBt/event_proportion‚0.02457682229578495 9"Purity/1FRt0SRqqt0FBt/event_purity‚0.0420084111392498 =&Purity/2FRt*SRqqt0FBt/event_proportion‚0.0382486991584301 @&Purity/2FRt*SRqqt1FBt/event_proportion‚0.0016043527284637094 ?&Purity/2FRt2SRqqt0FBt/event_proportion‚0.006417410913854837 particle/accuracy_1_of_4‚1 8 Purity/1FRt*SRqqt2FBt/FRt_purity‚0.48359790444374084 9"Purity/1FRt1SRqqt2FBt/SRqqt_purity‚0.2222222238779068 ("Purity/2FRt1SRqqt1FBt/event_purity‚0 ("Purity/2FRt*SRqqt2FBt/SRqqt_purity‚0 9"Purity/1FRt*SRqqt1FBt/SRqqt_purity‚0.4398128390312195 @&Purity/1FRt1SRqqt1FBt/event_proportion‚0.0064639137126505375 6validation_average_jet_accuracy‚0.1528697907924652 particle/accuracy_1_of_5‚1 8 Purity/2FRt*SRqqt*FBt/FRt_purity‚0.14360934495925903 9"Purity/*FRt*SRqqt1FBt/event_purity‚0.5830610990524292 7 Purity/*FRt2SRqqt0FBt/FRt_purity‚0.3269394338130951 :"Purity/0FR[ÄXY¡-t1SRqqt0FBt/event_purity‚0.13653510808944702 :"Purity/0FRt2SRqqt0FBt/SRqqt_purity‚0.15248341858386993 :"Purity/1FRt*SRqqt0FBt/SRqqt_purity‚0.12033935636281967 8&Purity/1FRt2SRqqt0FBt/event_proportion‚ 0.02294921875 ("Purity/2FRt2SRqqt0FBt/event_purity‚0 /particle/accuracy_3_of_4‚0.9943190813064575 8 Purity/*FRt*SRqqt0FBt/FRt_purity‚0.15686948597431183 ;"Purity/*FRt0SRqqt0FBt/event_purity‚0.040301259607076645 :"Purity/*FRt1SRqqt0FBt/event_purity‚0.07922850549221039 9 Purity/1FRt0SRqqt*FBt/FRt_purity‚0.049487482756376266 8 Purity/2FRt1SRqqt1FBt/FRt_purity‚0.19675926864147186 - Purity/2FRt0SRqqt1FBt/FRt_purity‚0.140625 & Purity/0FRt2SRqqt*FBt/FBt_purity‚1 >&Purity/0FRt2SRqqt0FBt/event_proportion‚0.02385602705180645 7 Purity/1FRt*SRqqt2FBt/FBt_purity‚0.9366118311882019 7 Purity/2FRt1SRqqt*FBt/FBt_purity‚0.9736841917037964 9"Purity/0FRt*SRqqt1FBt/SRqqt_purity‚0.3688167929649353 ?&Purity/0FRt*SRqqt2FBt/event_proportion‚0.008068266324698925 =&Purity/0FRt0SRqqt0FBt/event_proportion‚0.0462472103536129 <"Purity/2FRt*SRqqt0FBt/event_purity‚0.0011886304710060358 & Purity/2FRt0SRqqt*FBt/FBt_purity‚1 8 Purity/*FRt2SRqqt1FBt/FRt_purity‚0.20000000298023224 ,&Purity/0FRt2SRqqt2FBt/event_proportion‚0 ;"Purity/2FRt0SRqqt0FBt/event_purity‚0.002507915487512946 *particle/sensitivity‚0.517641007900238 7 Purity/*FRt*SRqqt1FBt/FRt_purity‚0.2790631949901581 :"Purity/*FRt2SRqqt1FBt/SRqqt_purity‚0.21759259700775146 8 Purity/1FRt*SRqqt*FBt/FRt_purity‚0.16366834938526154 ,&Purity/1FRt2SRqqt2FBt/event_proportion‚0 9 Purity/2FRt0SRqqt0FBt/FRt_purity‚0.030384119600057602 8 Purity/2FRt1SRqqt0FBt/FRt_purity‚0.29111894965171814 *jet/accuracy_1_of_4‚0.5856714248657227 /particle/accuracy_4_of_4‚0.9266747832298279 & Purity/*FRt2SRqqt*FBt/FBt_purity‚1 ?&Purity/0FRt0SRqqt2FBt/event_proportion‚0.007626487873494625 7 Purity/1FRt0SRqqt2FBt/FBt_purity‚0.9515212178230286 6 Purity/*FRt*SRqqt*FBt/FBt_purity‚0.969869077205658 9"Purity/1FRt*SRqqt2FBt/event_purity‚0.2630007565021515 @&Purity/1FRt*SRqqt2FBt/event_proportion‚0.0027204242069274187 , Purity/2FRt*SRqqt2FBt/FRt_purity‚0.21875 6 Purity/*FRt1SRqqt1FBt/FBt_purity‚0.986239492893219 :"Purity/1FRt1SRqqt1FBt/event_purity‚0.06484939903020859 jet/accuracy_4_of_4‚0 7 Purity/0FRt1SRqqt2FBt/FBt_purity‚0.6730769276618958 + Purity/2FRt*SRqqt2FBt/FBt_purity‚0.9375 @&Purity/*FRt1SRqqt2FBt/event_proportion‚0.0006975446594879031 @&Purity/*FRt2SRqqt1FBt/event_proportion‚0.0009998140158131719 9"Purity/0FRt*SRqqt2FBt/event_purity‚0.7942911982536316 >&Purity/1FRt0SRqqt1FBt/event_proportion‚0.00969587080180645 8 Purity/1FRt0SRqqt2FBt/FRt_purity‚0.47668179869651794 +jet/accuracy_1_of_2‚0.49586567282676697 =&Purity/*FRt1SRqqt*FBt/event_proportion‚0.3866024911403656 7 Purity/0FRt*SRqqt*FBt/FBt_purity‚0.9658235907554626 =&Purity/*FRt0SRqqt0FBt/event_proportion‚0.4607747495174408 9"Purity/0FRt0SRqqt2FBt/event_purity‚0.8411818146705627 :"Purity/*FRt*SRqqt0FBt/SRqqt_purity‚0.13012564182281494 :"Purity/*FRt*SRqqt2FBt/SRqqt_purity‚0.37037038803100586 7 Purity/*FRt0SRqqt1FBt/FBt_purity‚0.9879677891731262 =&Purity/0FRt*SRqqt*FBt/event_proportion‚0.2839471697807312 7 Purity/1FRt*SRqqt*FBt/FBt_purity‚0.9797457456588745 9"Purity/1FRt0SRqqt2FBt/event_purity‚0.2827286422252655 ?&Purity/2FRt1SRqqt0FBt/event_proportion‚0.011928013525903225 :"Purity/2FRt1SRqqt0FBt/SRqqt_purity‚0.08795001357793808 :"Purity/1FRt1SRqqt*FBt/SRqqt_purity‚0.12343824654817581 & Purity/2FRt0SRqqt2FBt/FBt_purity‚1 ( Purity/2FRt1SRqqt2FBt/FBt_purity‚0.5 trainer/global_step‚799 8 Purity/*FRt*SRqqt*FBt/FRt_purity‚0.16138596832752228 :"Purity/*FRt1SRqqt1FBt/SRqqt_purity‚0.41372230648994446 <&Purity/*FRt2SRqqt*FBt/event_proportion‚0.054222472012043 =&Purity/0FRt0SRqqt*FBt/event_proportion‚0.0390159972012043 8 Purity/1FRt*SRqqt1FBt/FRt_purity‚0.29709553718566895 7 Purity/1FRt0SRqqt0FBt/FRt_purity‚0.0420084111392498 7 Purity/1FRt1SRqqt1FBt/FBt_purity‚0.9970238208770752 =&Purity/2FRt0SRqqt*FBt/event_proportion‚0.0208565853536129 8 Purity/*FRt2SRqqt*FBt/FRt_purity‚0.32597336173057556 7 Purity/0FRt0SRqqt1FBt/FBt_purity‚0.9876819849014282 <"Purity/0FRt2SRqqt*FBt/event_purity‚0.0026392333675175905 <&Purity/2FRt*SRqqt*FBt/event_proportion‚0.040062315762043 +jet/accuracy_2_of_4‚0.19597773253917694 particle/accuracy_3_of_5‚1 <"Purity/*FRt2SRqqt0FBt/event_purity‚0.0012198289623484015 9"Purity/0FRt2SRqqt*FBt/SRqqt_purity‚0.1535085290670395 A&Purity/1FRt2SRqqt1FBt/event_proportion‚0.00027901786961592734 8 Purity/2FRt*SRqqt0FBt/FRt_purity‚0.14210207760334015 +jet/accuracy_2_of_3‚0.18000510334968567 :"Purity/0FRt1SRqqt2FBt/SRqqt_purity‚0.42307692766189575 ("Purity/2FRt0SRqqt2FBt/event_purity‚0 jet/accuracy_4_of_5‚0 9"Purity/0FRt*SRqqt0FBt/SRqqt_purity‚0.1394970864057541 ;"Purity/1FRt1SRqqt*FBt/event_purity‚0.006689449772238731 9"Purity/2FRt*SRqqt1FBt/SRqqt_purity‚0.5277777910232544 ) Purity/2FRt0SRqqt2FBt/FRt_purity‚0.25 & Purity/2FRt1SRqqt2FBt/FRt_purity‚0 particle/accuracy_2_of_5‚1 *loss/val_total_loss‚24.535247402953072 6 Purity/0FRt*SRqqt2FBt/FBt_purity‚0.907890796661377 8 Purity/2FRt*SRqqt1FBt/FRt_purity‚0.18166665732860565 ("Purity/2FRt1SRqqt2FBt/SRqqt_purity‚0 @&Purity/0FRt2SRqqt1FBt/event_proportion‚0.0007207961170934141 :"Purity/1FRt1SRqqt0FBt/SRqqt_purity‚0.10884857177734375 :"Purity/2FRt*SRqqt*FBt/SRqqt_purity‚0.12110118567943573 jet/accuracy_5_of_5‚0 7 Purity/0FRt1SRqqt*FBt/FBt_purity‚0.9615462422370911 <"Purity/*FRt2SRqqt*FBt/event_purity‚0.0012034674873575568 8 Purity/*FRt*SRqqt2FBt/FRt_purity‚0.45699164271354675 6 Purity/2FRt*SRqqt*FBt/FBt_purity‚0.987500011920929 *jet/accuracy_1_of_1‚0.1046723946928978 ("Purity/*FRt1SRqqt2FBt/event_purity‚0 9"Purity/1FRt2SRqqt*FBt/SRqqt_purity‚0.1572919636964798 particle/accuracy_1_of_3‚1 jet/accuracy_3_of_5‚0 =&Purity/0FRt*SRqqt0FBt/event_proportion‚0.2322126179933548 7 Purity/1FRt0SRqqt*FBt/FBt_purity‚0.9776633381843567 ("Purity/1FRt2SRqqt*FBt/event_purity‚0 +"Purity/1FRt2SRqqt1FBt/SRqqt_purity‚0.25 +particle/specificity‚0.8781355619430542 7 Purity/*FRt*SRqqt2FBt/FBt_purity‚0.9160813689231873 :"Purity/*FRt2SRqqt*FBt/SRqqt_purity‚0.15218552947044373 :"Purity/1FRt0SRqqt*FBt/event_purity‚0.04672415554523468 =&Purity/1FRt1SRqqt0FBt/event_proportion‚0.1467633992433548 8 Purity/1FRt*SRqqt0FBt/FRt_purity‚0.15871678292751312 7 Purity/1FRt0SRqqt1FBt/FRt_purity‚0.2803121507167816 ("Purity/1FRt2SRqqt0FBt/event_purity‚0 <"Purity/2FRt*SRqqt*FBt/event_purity‚0.0018069216748699546 8 Purity/2FRt0SRqqt*FBt/FRt_purity‚0.03666163235902786 8 Purity/2FRt1SRqqt*FBt/FRt_purity‚0.28315430879592896 /particle/accuracy_3_of_3‚0.9985816478729248 (particle/accuracy‚0.8007115125656128 9"Purity/0FRt0SRqqt*FBt/event_purity‚0.9596776366233826 :"Purity/0FRt1SRqqt1FBt/event_purity‚0.22036917507648468 ("Purity/0FRt2SRqqt1FBt/event_purity‚0 @&Purity/1FRt1SRqqt2FBt/event_proportion‚0.0002325148816453293 & Purity/1FRt2SRqqt1FBt/FBt_purity‚1 8 Purity/*FRt0SRqqt0FBt/FRt_purity‚0.04115373641252518 9"Purity/*FRt0SRqqt1FBt/event_purity‚0.7894789576530457 :"Purity/0FRt*SRqqt*FBt/event_purity‚0.24124176800251007 =&Purity/1FRt*SRqqt*FBt/event_proportion‚0.6297432780265808 :"Purity/1FRt*SRqqt*FBt/SRqqt_purity‚0.13164475560188293 >&Purity/1FRt2SRqqt*FBt/event_proportion‚0.02322823740541935 particle/accuracy_2_of_2‚1 particle/accuracy_4_of_5‚0 ("Purity/*FRt2SRqqt1FBt/event_purity‚0 9"Purity/1FRt*SRqqt2FBt/SRqqt_purity‚0.2222222238779068 & Purity/1FRt2SRqqt*FBt/FBt_purity‚1 (jet/accuracy_2_of_2‚0.05192856118083 :"Purity/2FRt0SRqqt1FBt/event_purity‚0.02083333395421505 :"Purity/*FRt*SRqqt*FBt/SRqqt_purity‚0.14161144196987152 7 Purity/1FRt0SRqqt1FBt/FBt_purity‚0.9872298240661621 8 Purity/1FRt1SRqqt0FBt/FRt_purity‚0.47323599457740784 :"Purity/*FRt0SRqqt*FBt/event_purity‚0.11433357000350952 ( Purity/*FRt1SRqqt2FBt/FRt_purity‚0.5 8 Purity/1FRt2SRqqt1FBt/FRt_purity‚0.20000000298023224 :"Purity/2FRt1SRqqt*FBt/SRqqt_purity‚0.11689165979623795 jet/accuracy_1_of_5‚1 @&Purity/2FRt1SRqqt1FBt/event_proportion‚0.0008370535797439516 7 Purity/*FRt1SRqqt2FBt/FBt_purity‚0.6898148059844971 7 Purity/0FRt1SRqqt1FBt/FBt_purity‚0.9805048704147339 @&Purity/0FRt1SRqqt2FBt/event_proportion‚0.0004417782765813172 9"Purity/1FRt1SRqqt1FBt/SRqqt_purity‚0.4492231607437134 & Purity/2FRt*SRqqt1FBt/FBt_purity‚1 @&Purity/2FRt*SRqqt2FBt/event_proportion‚0.0002092633949359879 ("Purity/2FRt1SRqqt*FBt/event_purity‚0 :"Purity/*FRt*SRqqt*FBt/event_purity‚0.09522418677806854 =&Purity/*FRt*SRqqt*FBt/event_proportion‚0.9537528157234192 =&Purity/*FRt0SRqqt*FBt/event_proportion‚0.5129278302192688 particle/accuracy_1_of_2‚1 particle/accuracy_5_of_5‚0 epoch‚1 % _timestamp‚1.7844608732282608e+09 =&Purity/1FRt0SRqqt*FBt/event_proportion‚0.4530552327632904 A&Purity/2FRt0SRqqt2FBt/event_proportion‚0.00018601190822664648 :"Purity/0FRt*SRqqt*FBt/SRqqt_purity‚0.15096595883369446 :"Purity/1FRt*SRqqt*FBt/event_purity‚0.03526797890663147 ,&Purity/2FRt2SRqqt2FBt/event_proportion‚0 :"Purity/0FRt*SRqqt2FBt/SRqqt_purity‚0.42307692766189575 ("Purity/0FRt0SRqqt0FBt/event_purity‚1 =&Purity/0FRt1SRqqt*FBt/event_proportion‚0.2203543484210968 =&Purity/0FRt1SRqqt0FBt/event_proportion‚0.2083565890789032 :"Purity/0FRt1SRqqt0FBt/SRqqt_purity‚0.13653510808944702 <"Purity/0FRt2SRqqt0FBt/event_purity‚0.0026627304032444954 :"Purity/1FRt*SRqqt0FBt/event_purity‚0.03135574609041214 =&Purity/2FRt1SRqqt*FBt/event_proportion‚0.0127883180975914 :"Purity/2FRt2SRqqt0FBt/SRqqt_purity‚0.12185085564851761 :"Purity/*FRt1SRqqt0FBt/SRqqt_purity‚0.12397666275501251 9"Purity/2FRt1SRqqt1FBt/SRqqt_purity‚0.5277777910232544 particle/accuracy_2_of_4‚1 :"Purity/0FRt1SRqqt*FBt/SRqqt_purity‚0.15040554106235504 6"Purity/*FRt1SRqqt*FBt/SRqqt_purity‚0.1385488063097 :"Purity/1FRt*SRqqt1FBt/event_purity‚0.15048959851264954 ,jet/accuracy_3_of_4‚0.013155089691281319 7 Purity/*FRt1SRqqt*FBt/FRt_purity‚0.4406296908855438 ,&Purity/*FRt2SRqqt2FBt/event_proportion‚0 =&Purity/0FRt0SRqqt1FBt/event_proportion‚0.0313895083963871 ?&Purity/0FRt1SRqqt1FBt/event_proportion‚0.011555989272892475 ("Purity/1FRt1SRqqt2FBt/event_purity‚0 :"Purity/2FRt*SRqqt0FBt/SRqqt_purity‚0.10816771537065506 :"Purity/2FRt2SRqqt*FBt/SRqqt_purity‚0.12185085564851761 ("Purity/2FRt*SRqqt2FBt/event_purity‚0 :"Purity/*FRt*SRqqt1FBt/SRqqt_purity‚0.39528724551200867 7 Purity/*FRt0SRqqt1FBt/FRt_purity‚0.2660948634147644 7 Purity/1FRt1SRqqt*FBt/FBt_purity‚0.9849583506584167 ("Purity/2FRt2SRqqt*FBt/event_purity‚0 7 Purity/2FRt2SRqqt*FBt/FRt_purity‚0.2159012109041214 :"Purity/*FRt1SRqqt*FBt/event_purity‚0.08298203349113464 =&Purity/*FRt1SRqqt0FBt/event_proportion‚0.3670479953289032 7 Purity/0FRt0SRqqt*FBt/FBt_purity‚0.9663628935813904 9"Purity/0FRt1SRqqt*FBt/event_purity‚0.1407722681760788 =&Purity/1FRt1SRqqt*FBt/event_proportion‚0.1534598171710968 ?&Purity/*FRt*SRqqt2FBt/event_proportion‚0.010997953824698925 7 Purity/*FRt0SRqqt*FBt/FBt_purity‚0.9693504571914673 >&Purity/1FRt*SRqqt1FBt/event_proportion‚0.01643880270421505 7 Purity/1FRt*SRqqt1FBt/FBt_purity‚0.9918235540390015 7 Purity/1FRt1SRqqt1FBt/FRt_purity‚0.3225094974040985 ,&Purity/2FRt2SRqqt1FBt/event_proportion‚0 'particle/f_score‚0.5273159742355347Ê'=Â>Ì‹| 6 Purity/*FRt1SRqqt1FBt/FBt_purity‚0.986239492893219 :"Purity/1FRt1SRqqt1FBt/event_purity‚0.06484939903020859 jet/accuracy_4_of_4‚0 7 Purity/0FRt1SRqqt2FBt/FBt_purity‚0.6730769276618958 + Purity/2FRt*SRqqt2FBt/FBt_purity‚0.9375 @&Purity/*FRt1SRqqt2FBt/event_proportion‚0.0006975446594879031 @&Purity/*FRt2SRqqt1FBt/event_proportion‚0.0009998140158131719 9"Purity/0FRt*SRqqt2FBt/event_purity‚0.7942911982536316 >&Purity/1FRt0SRqqt1FBt/event_proportion‚0.00969587080180645 8 Purity/1FRt0SRqqt2FBt/FRt_purity‚0.47668179869651794 +jet/accuracy_1_of_2‚0.49586567282676697 =&Purity/*FRt1SRqqt*FBt/event_proportion‚0.3866024911403656 7 Purity/0FRt*SRqqt*FBt/FBt_purity‚0.9658235907554626 =&Purity/*FRt0SRqqt0FBt/event_proportion‚0.4607747495174408 9"Purity/0FRt0SRqqt2FBt/event_purity‚0.8411818146705627 :"Purity/*FRt*SRqqt0FBt/SRqqt_purity‚0.13012564182281494 :"Purity/*FRt*SRqqt2FBt/SRqqt_purity‚0.37037038803100586 7 Purity/*FRt0SRqqt1FBt/FBt_purity‚0.9879677891731262 =&Purity/0FRt*SRqqt*FBt/event_proportion‚0.2839471697807312 7 Purity/1FRt*SRqqt*FBt/FBt_purity‚0.9797457456588745 9"Purity/1FRt0SRqqt2FBt/event_purity‚0.2827286422252655 ?&Purity/2FRt1SRqqt0FBt/event_proportion‚0.011928013525903225 :"Purity/2FRt1SRqqt0FBt/SRqqt_purity‚0.08795001357793808 :"Purity/1FRt1SRqqt*FBt/SRqqt_purity‚0.12343824654817581 & Purity/2FRt0SRqqt2FBt/FBt_purity‚1 ( Purity/2FRt1SRqqt2FBt/FBt_purity‚0.5 trainer/global_step‚799 8 Purity/*FRt*SRqqt*FBt/FRt_purity‚0.16138596832752228 :"Purity/*FRt1SRqqt1FBt/SRqqt_purity‚0.41372230648994446 <&Purity/*FRt2SRqqt*FBt/event_proportion‚0.054222472012043 =&Purity/0FRt0SRqqt*FBt/event_proportion‚0.0390159972012043 8 Purity/1FRt*SRqqt1FBt/FRt_purity‚0.29709553718566895 7 Purity/1FRt0SRqqt0FBt/FRt_purity‚0.0420084111392498 7 Purity/1FRt1SRqqt1FBt/FBt_purity‚0.9970238208770752 =&Purity/2FRt0SRqqt*FBt/event_proportion‚0.0208565853536129 8 Purity/*FRt2SRqqt*FBt/FRt_purity‚0.32597336173057556 7 Purity/0FRt0SRqqt1FBt/FBt_purity‚0.9876819849014282 <"Purity/0FRt2SRqqt*FBt/event_purity‚0.0026392333675175905 <&Purity/2FRt*SRqqt*FBt/event_proportion‚0.040062315762043 +jet/accuracy_2_of_4‚0.19597773253917694 particle/accuracy_3_of_5‚1 <"Purity/*FRt2SRqqt0FBt/event_purity‚0.0012198289623484015 9"Purity/0FRt2SRqqt*FBt/SRqqt_purity‚0.1535085290670395 A&Purity/1FRt2SRqqt1FBt/event_proportion‚0.00027901786961592734 8 Purity/2FRt*SRqqt0FBt/FRt_purity‚0.14210207760334015 +jet/accuracy_2_of_3‚0.18000510334968567 :"Purity/0FRt1SRqqt2FBt/SRqqt_purity‚0.42307692766189575 ("Purity/2FRt0SRqqt2FBt/event_purity‚0 jet/accuracy_4_of_5‚0 9"Purity/0FRt*SRqqt0FBt/SRqqt_purity‚0.1394970864057541 ;"Purity/1FRt1SRqqt*FBt/event_purity‚0.006689449772238731 9"Purity/2FRt*SRqqt1FBt/SRqqt_purity‚0.5277777910232544 ) Purity/2FRt0SRqqt2FBt/FRt_purity‚0.25 & Purity/2FRt1SRqqt2FBt/FRt_purity‚0 particle/accuracy_2_of_5‚1 *loss/val_total_loss‚24.535247402953072 6 Purity/0FRt*SRqqt2FBt/FBt_purity‚0.907890796661377 8 Purity/2FRt*SRqqt1FBt/FRt_purity‚0.18166665732860565 ("Purity/2FRt1SRqqt2FBt/SRqqt_purity‚0 @&Purity/0FRt2SRqqt1FBt/event_proportion‚0.0007207961170934141 :"Purity/1FRt1SRqqt0FBt/SRqqt_purity‚0.10884857177734375 :"Purity/2FRt*SRqqt*FBt/SRqqt_purity‚0.12110118567943573 jet/accuracy_5_of_5‚0 7 Purity/0FRt1SRqqt*FBt/FBt_purity‚0.9615462422370911 <"Purity/*FRt2SRqqt*FBt/event_purity‚0.0012034674873575568 8 Purity/*FRt*SRqqt2FBt/FRt_purity‚0.45699164271354675 6 Purity/2FRt*SRqqt*FBt/FBt_purity‚0.987500011920929 *jet/accuracy_1_of_1‚0.1046723946928978 ("Purity/*FRt1SRqqt2FBt/event_purity‚0 9"Purity/1FRt2SRqqt*FBt/SRqqt_purity‚0.1572919636964798 particle/accuracy_1_of_3‚1 jet/accuracy_3_of_5‚0 =&Purity/0FRt*SRqqt0FBt/event_proportion‚0.2322126179933548 7 Purity/1FRt0SRqqt*FBt/FBt_purity‚0.9776633381843567 ("Purity/1FRt2SRqqt*FBt/event_purity‚0 +"Purity/1FRt2SRqqt1FBt/SRqqt_purity‚0.25 +particle/specificity‚0.8781355619430542 7 Purity/*FRt*SRqqt2FBt/FBt_purity‚0.9160813689231873 :"Purity/*FRt2SRqqt*FBt/SRqqt_purity‚0.15218552947044373 :"Purity/1FRt0SRqqt*FBt/event_purity‚0.04672415554523468 =&Purity/1FRt1SRqqt0FBt/event_proportion‚0.1467633992433548 8 Purity/1FRt*SRqqt0FBt/FRt_purity‚0.15871678292751312 7 Purity/1FRt0SRqqt1FBt/FRt_purity‚0.2803121507167816 ("Purity/1FRt2SRqqt0FBt/event_purity‚0 <"Purity/2FRt*SRqqt*FBt/event_purity‚0.0018069216748699546 8 Purity/2FRt0SRqqt*FBt/FRt_purity‚0.03666163235902786 8 Purity/2FRt1SRqqt*FBt/FRt_purity‚0.28315430879592896 /particle/accuracy_3_of_3‚0.9985816478729248 (particle/accuracy‚0.8007115125656128 9"Purity/0FRt0SRqqt*FBt/event_purity‚0.9596776366233826 :"Purity/0FRt1SRqqt1FBt/event_purity‚0.22036917507648468 ("Purity/0FRt2SRqqt1FBt/event_purity‚0 @&Purity/1FRt1SRqqt2FBt/event_proportion‚0.0002325148816453293 & Purity/1FRt2SRqqt1FBt/FBt_purity‚1 8 Purity/*FRt0SRqqt0FBt/FRt_purity‚0.04115373641252518 9"Purity/*FRt0SRqqt1FBt/event_purity‚0.7894789576530457 :"Purity/0FRt*SRqqt*FBt/event_purity‚0.24124176800251007 =&Purity/1FRt*SRqqt*FBt/event_proportion‚0.6297432780265808 :"Purity/1FRt*SRqqt*FBt/SRqqt_purity‚0.13164475560188293 >&Purity/1FRt2SRqqt*FBt/event_proportion‚0.02322823740541935 particle/accuracy_2_of_2‚1 particle/accuracy_4_of_5‚0 ("Purity/*FRt2SRqqt1FBt/event_purity‚0 9"Purity/1FRt*SRqqt2FBt/SRqqt_purity‚0.2222222238779068 & Purity/1FRt2SRqqt*FBt/FBt_purity‚1 (jet/accuracy_2_of_2‚0.05192856118083 :"Purity/2FRt0SRqqt1FBt/event_purity‚0.02083333395421505 :"Purity/*FRt*SRqqt*FBt/SRqqt_purity‚0.14161144196987152 7 Purity/1FRt0SRqqt1FBt/FBt_purity‚0.9872298240661621 8 Purity/1FRt1SRqqt0FBt/FRt_purity‚0.47323599457740784 :"Purity/*FRt0SRqqt*FBt/event_purity‚0.11433357000350952 ( Purity/*FRt1SRqqt2FBt/FRt_purity‚0.5 8 Purity/1FRt2SRqqt1FBt/FRt_purity‚0.20000000298023224 :"Purity/2FRt1SRqqt*FBt/SRqqt_purity‚0.11689165979623795 jet/accuracy_1_of_5‚1 @&Purity/2FRt1SRqqt1FBt/event_proportion‚0.0008370535797439516 7 Purity/*FRt1SRqqt2FBt/FBt_purity‚0.6898148059844971 7 Purity/0FRt1SRqqt1FBt/FBt_purity‚0.9805048704147339 @&Purity/0FRt1SRqqt2FBt/event_proportion‚0.0004417782765813172 9"Purity/1FRt1SRqqt1FBt/SRqqt_purity‚0.4492231607437134 & Purity/2FRt*SRqqt1FBt/FBt_purity‚1 @&Purity/2FRt*SRqqt2FBt/event_proportion‚0.0002092633949359879 ("Purity/2FRt1SRqqt*FBt/event_purity‚0 :"Purity/*FRt*SRqqt*FBt/event_purity‚0.09522418677806854 =&Purity/*FRt*SRqqt*FBt/event_proportion‚0.9537528157234192 =&Purity/*FRt0SRqqt*FBt/event_proportion‚0.5129278302192688 particle/accuracy_1_of_2‚1 particle/accuracy_5_of_5‚0 epoch‚1 % _timestamp‚1.7844608732282608e+09 =&Purity/1FRt0SRqqt*FBt/event_proportion‚0.4530552327632904 A&Purity/2FRt0SRqqt2FBt/event_proportion‚0.00018601190822664648 :"Purity/0FRt*SRqqt*FBt/SRqqt_purity‚0.15096595883369446 :"Purity/1FRt*SRqqt*FBt/event_purity‚0.03526797890663147 ,&Purity/2FRt2SRqqt2FBt/event_proportion‚0 :"Purity/0FRt*SRqqt2FBt/SRqqt_purity‚0.42307692766189575 ("Purity/0FRt0SRqqt0FBt/event_purity‚1 =&Purity/0FRt1SRqqt*FBt/event_proportion‚0.2203543484210968 =&Purity/0FRt1SRqqt0FBt/event_proportion‚0.2083565890789032 :"Purity/0FRt1SRqqt0FBt/SRqqt_purity‚0.13653510808944702 <"Purity/0FRt2SRqqt0FBt/event_purity‚0.0026627304032444954 :"Purity/1FRt*SRqqt0FBt/event_purity‚0.03135574609041214 =&Purity/2FRt1SRqqt*FBt/event_proportion‚0.0127883180975914 :"Purity/2FRt2SRqqt0FBt/SRqqt_purity‚0.12185085564851761 :"Purity/*FRt1SRqqt0FBt/SRqqt_purity‚0.12397666275501251 9"Purity/2FRt1SRqqt1FBt/SRqqt_purity‚0.5277777910232544 particle/accuracy_2_of_4‚1 :"Purity/0FRt1SRqqt*FBt/SRqqt_purity‚0.15040554106235504 6"Purity/*FRt1SRqqt*FBt/SRqqt_purity‚0.1385488063097 :"Purity/1FRt*SRqqt1FBt/event_purity‚0.15048959851264954 ,jet/accuracy_3_of_4‚0.013155089691281319 7 Purity/*FRt1SRqqt*FBt/FRt_purity‚0.4406296908855438 ,&Purity/*FRt2SRqqt2FBt/event_proportion‚0 =&Purity/0FRt0SRqqt1FBt/event_proportion‚0.0313895083963871 ?&Purity/0FRt1SRqqt1FBt/event_proportion‚0.011555989272892475 ("Purity/1FRt1SRqqt2FBt/event_purity‚0 :"Purity/2FRt*SRqqt0FBt/SRqqt_purity‚0.10816771537065506 :"Purity/2FRt2SRqqt*FBt/SRqqt_purity‚0.12185085564851761 ("Purity/2FRt*SRqqt2FBt/event_purity‚0 :"Purity/*FRt*SRqqt1FBt/SRqqt_purity‚0.39528724551200867 7 Purity/*FRt0SRqqt1FBt/FRt_purity‚0.2660948634147644 7 Purity/1FRt1SRqqt*FBt/FBt_purity‚0.9849583506584167 ("Purity/2FRt2SRqqt*FBt/event_purity‚0 7 Purity/2FRt2SRqqt*FBt/FRt_purity‚0.2159012109041214 :"Purity/*FRt1SRqqt*FBt/event_purity‚0.08298203349113464 =&Purity/*FRt1SRqqt0FBt/event_proportion‚0.3670479953289032 7 Purity/0FRt0SRqqt*FBt/FBt_purity‚0.9663628935813904 9"Purity/0FRt1SRqqt*FBt/event_purity‚0.1407722681760788 =&Purity/1FRt1SRqqt*FBt/event_proportion‚0.1534598171710968 ?&Purity/*FRt*SRqqt2FBt/event_proportion‚0.010997953824698925 7 Purity/*FRt0SRqqt*FBt/FBt_purity‚0.9693504571914673 >&Purity/1FRt*SRqqt1FBt/event_proportion‚0.01643880270421505 7 Purity/1FRt*SRqqt1FBt/FBt_purity‚0.9918235540390015 7 Purity/1FRt1SRqqt1FBt/FRt_purity‚0.3225094974040985 ,&Purity/2FRt2SRqqt1FBt/event_proportion‚0 'particle/f_score‚0.5273159742355347 =&Purity/*FRt0SRqqt2FBt/event_proportion‚0.0103004090487957 9"Purity/0FRt*SRqqt1FBt/event_purity‚0.7674689292907715 8 Purity/1FRt2SRqqt0FBt/FRt_purity‚0.39152753353118896 ,jet/accuracy_3_of_3‚0.019393548369407654 8 Purity/*FRt1SRqqt1FBt/FRt_purity‚0.29602086544036865 7 Purity/0FRt*SRqqt1FBt/FBt_purity‚0.9865090250968933 9"Purity/0FRt0SRqqt1FBt/event_purity‚0.9876819849014282 @&Purity/1FRt0SRqqt2FBt/event_proportion‚0.0024879092816263437 :"Purity/*FRt1SRqqt2FBt/SRqqt_purity‚0.37037038803100586 =&Purity/1FRt*SRqqt0FBt/event_proportion‚0.6105840802192688 =&Purity/*FRt0SRqqt1FBt/event_proportion‚0.0418526791036129 7 Purity/*FRt1SRqqt*FBt/FBt_purity‚0.9683840870857239 :"Purity/1FRt2SRqqt0FBt/SRqqt_purity‚0.15531933307647705 ?&Purity/2FRt2SRqqt*FBt/event_proportion‚0.006417410913854837 ("Purity/2FRt1SRqqt2FBt/event_purity‚0 _runtime‚ 547.617006062 =&Purity/*FRt1SRqqt1FBt/event_proportion‚0.0188569575548172 & Purity/*FRt2SRqqt1FBt/FBt_purity‚1 ("Purity/1FRt2SRqqt1FBt/event_purity‚0 @&Purity/2FRt1SRqqt2FBt/event_proportion‚2.325148852833081e-05 *jet/accuracy_1_of_3‚0.6444879174232483 7 Purity/*FRt*SRqqt1FBt/FBt_purity‚0.9882218241691589 =&Purity/1FRt0SRqqt0FBt/event_proportion‚0.4408714771270752 7 Purity/1FRt1SRqqt*FBt/FRt_purity‚0.4671282172203064 ;"Purity/1FRt1SRqqt0FBt/event_purity‚0.004130990710109472 >&Purity/2FRt0SRqqt0FBt/event_proportion‚0.01990327425301075 =&Purity/*FRt*SRqqt1FBt/event_proportion‚0.0617094486951828 =&Purity/0FRt*SRqqt1FBt/event_proportion‚0.0436662957072258 :"Purity/1FRt0SRqqt1FBt/event_purity‚0.20962484180927277 <&Purity/*FRt*SRqqt0FBt/event_proportion‚0.881045401096344 9 Purity/*FRt0SRqqt*FBt/FRt_purity‚0.048534296452999115 :"Purity/0FRt1SRqqt1FBt/SRqqt_purity‚0.38826343417167664 -"Purity/0FRt2SRqqt1FBt/SRqqt_purity‚0.1875 @&Purity/2FRt0SRqqt1FBt/event_proportion‚0.0007672990905120969 6 Purity/*FRt0SRqqt2FBt/FBt_purity‚0.929855465888977 :"Purity/*FRt2SRqqt0FBt/SRqqt_purity‚0.15093088150024414 & Purity/0FRt2SRqqt1FBt/FBt_purity‚1 7 Purity/1FRt2SRqqt*FBt/FRt_purity‚0.3886665999889374 & Purity/2FRt1SRqqt1FBt/FBt_purity‚1 & Purity/2FRt0SRqqt1FBt/FBt_purity‚1 9"Purity/*FRt*SRqqt2FBt/event_purity‚0.6427225470542908 :"Purity/*FRt1SRqqt1FBt/event_purity‚0.15715990960597992 :"Purity/0FRt*SRqqt0FBt/event_purity‚0.12270773947238922 7 Purity/1FRt1SRqqt2FBt/FBt_purity‚0.7777777910232544 9"Purity/*FRt0SRqqt2FBt/event_purity‚0.6868665814399719 8&Purity/*FRt2SRqqt0FBt/event_proportion‚ 0.05322265625 7 Purity/0FRt0SRqqt2FBt/FBt_purity‚0.9205908179283142 7 Purity/2FRt2SRqqt0FBt/FRt_purity‚0.2159012109041214 jet/accuracy_2_of_5‚0 ;"Purity/*FRt*SRqqt0FBt/event_purity‚0.054136067628860474 8 Purity/*FRt1SRqqt0FBt/FRt_purity‚0.44761916995048523 <"Purity/2FRt0SRqqt*FBt/event_purity‚0.0036614774726331234 ("Purity/2FRt1SRqqt0FBt/event_purity‚0 7 Purity/1FRt1SRqqt2FBt/FRt_purity‚0.5555555820465088 particle/accuracy_1_of_1‚1 _step‚65 ;"Purity/2FRt*SRqqt1FBt/event_purity‚0.009999999776482582 particle/accuracy_2_of_3‚1 ("Purity/0FRt1SRqqt2FBt/event_purity‚0 8 Purity/*FRt0SRqqt2FBt/FRt_purity‚0.45891907811164856 >&Purity/0FRt2SRqqt*FBt/event_proportion‚0.02457682229578495 9"Purity/1FRt0SRqqt0FBt/event_purity‚0.0420084111392498 =&Purity/2FRt*SRqqt0FBt/event_proportion‚0.0382486991584301 @&Purity/2FRt*SRqqt1FBt/event_proportion‚0.0016043527284637094 ?&Purity/2FRt2SRqqt0FBt/event_proportion‚0.006417410913854837 particle/accuracy_1_of_4‚1 8 Purity/1FRt*SRqqt2FBt/FRt_purity‚0.48359790444374084 9"Purity/1FRt1SRqqt2FBt/SRqqt_purity‚0.2222222238779068 ("Purity/2FRt1SRqqt1FBt/event_purity‚0 ("Purity/2FRt*SRqqt2FBt/SRqqt_purity‚0 9"Purity/1FRt*SRqqt1FBt/SRqqt_purity‚0.4398128390312195 @&Purity/1FRt1SRqqt1FBt/event_proportion‚0.0064639137126505375 6validation_average_jet_accuracy‚0.1528697907924652 particle/accuracy_1_of_5‚1 8 Purity/2FRt*SRqqt*FBt/FRt_purity‚0.14360934495925903 9"Purity/*FRt*SRqqt1FBt/event_purity‚0.5830610990524292 7 Purity/*FRt2SRqqt0FBt/FRt_purity‚0.3269394338130951 :"Purity/0FRt1SRqqt0FBt/event_purity‚0.13653510808944702 :"Purity/0FRt2SRqqt0FBt/SRqqt_purity‚0.15248341858386993 :"Purity/1FRt*SRqqt0FBt/SRqqt_purity‚0.12033935636281967 8&Purity/1FRt2SRqqt0FBt/event_proportion‚ 0.02294921875 ("Purity/2FRt2SRqqt0FBt/event_purity‚0 /particle/accuracy_3_of_4‚0.9943190813064575 8 Purity/*FRt*SRqqt0FBt/FRt_purity‚0.15686948597431183 ;"Purity/*FRt0SRqqt0FBt/event_purity‚0.040301259607076645 :"Purity/*FRt1SRqqt0FBt/event_purity‚0.07922850549221039 9 Purity/1FRt0SRqqt*FBt/FRt_purity‚0.049487482756376266 8 Purity/2FRt1SRqqt1FBt/FRt_purity‚0.19675926864147186 - Purity/2FRt0SRqqt1FBt/FRt_purity‚0.140625 & Purity/0FRt2SRqqt*FBt/FBt_purity‚1 >&Purity/0FRt2SRqqt0FBt/event_proportion‚0.02385602705180645 7 Purity/1FRt*SRqqt2FBt/FBt_purity‚0.9366118311882019 7 Purity/2FRt1SRqqt*FBt/FBt_purity‚0.9736841917037964 9"Purity/0FRt*SRqqt1FBt/SRqqt_purity‚0.3688167929649353 ?&Purity/0FRt*SRqqt2FBt/event_proportion‚0.008068266324698925 =&Purity/0FRt0SRqqt0FBt/event_proportion‚0.0462472103536129 <"Purity/2FRt*SRqqt0FBt/event_purity‚0.0011886304710060358 & Purity/2FRt0SRqqt*FBt/FBt_purity‚1 8 Purity/*FRt2SRqqt1FBt/FRt_purity‚0.20000000298023224 ,&Purity/0FRt2SRqqt2FBt/event_proportion‚0 ;"Purity/2FRt0SRqqt0FBt/event_purity‚0.002507915487512946 *particle/sensitivity‚0.517641007900238 7 Purity/*FRt*SRqqt1FBt/FRt_purity‚0.2790631949901581 :"Purity/*FRt2SRqqt1FBt/SRqqt_purity‚0.21759259700775146 8 Purity/1FRt*SRqqt*FBt/FRt_purity‚0.16366834938526154 ,&Purity/1FRt2SRqqt2FBt/event_proportion‚0 9 Purity/2FRt0SRqqt0FBt/FRt_purity‚0.030384119600057602 8 Purity/2FRt1SRqqt0FBt/FRt_purity‚0.29111894965171814 *jet/accuracy_1_of_4‚0.5856714248657227 /particle/accuracy_4_of_4‚0.9266747832298279 & Purity/*FRt2SRqqt*FBt/FBt_purity‚1 ?&Purity/0FRt0SRqqt2FBt/event_proportion‚0.007626487873494625 7 Purity/1FRt0SRqqt2FBt/FBt_purity‚0.9515212178230286 6 Purity/*FRt*SRqqt*FBt/FBt_purity‚0.969869077205658 9"Purity/1FRt*SRqqt2FBt/event_purity‚0.2630007565021515 @&Purity/1FRt*SRqqt2FBt/event_proportion‚0.0027204242069274187 , Purity/2FRt*SRqqt2FBt/FRt_purity‚0.21875AòT¨©mÍ‚(:â ÐìòÒ†‚¼¥ proc.memory.rssMB‚ 2827.5625 cpu‚25.791380708437035 network.recv‚190884 disk.nvme1n1.in‚ 129.765625& disk.nvme1n1.out‚10777.64501953125 disk.rbd0.out‚0' proc.memory.availableMB‚ 128.2578125* proc.memory.percent‚17.258071899414062 proc.cpu.threads‚17 network.sent‚ 1.08918e+06) disk./.usagePercent‚76.45102188614754( disk.nvme1c1n1.out‚10777.64501953125 disk.md0.out‚ 15.703125$ memory_percent‚99.21717643737793# disk./.usageGB‚1173.71431350708" disk.nvme0c0n1.in‚ 1852.4453125% disk.nvme0n1.out‚14538.4990234375 disk.nvme1c1n1.in‚ 129.765625 disk.md0.in‚0 disk.rbd0.in‚0' disk.nvme0c0n1.out‚14538.4990234375 disk.nvme0n1.in‚ 1852.4453125 disk.dm-0.in‚ 1982.2109375" disk.dm-0.out‚24561.8916015625 disk.dm-1.in‚0! disk.dm-1.out‚720.709472656256í.hŸÎ‚(:” ÐìòÒ€Œü¤ gpu.0.gpu‚92.0 gpu.0.memory‚19, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚49.0 gpu.0.powerWatts‚115.302' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚38.434000000000005 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0}~çtÏ‚(:é ßìòÒï­ô¤" disk.nvme0c0n1.in‚ 1852.4453125# memory_percent‚99.2121934890747( proc.memory.availableMB‚ 129.07421875 network.sent‚ 1.132675e+06 disk.nvme1c1n1.in‚ 129.765625( disk.nvme1c1n1.out‚10841.99267578125& disk.nvme1n1.out‚10841.99267578125" disk.dm-0.out‚24562.4736328125 disk.dm-1.in‚0 disk.rbd0.in‚0' disk.nvme0c0n1.out‚14539.2392578125) disk./.usagePercent‚76.45089764914165% disk.nvme0n1.out‚14539.2392578125 disk.nvme1n1.in‚ 129.765625 disk.md0.in‚0 disk.md0.out‚ 15.95703125 disk.rbd0.out‚0! proc.memory.rssMB‚ 2826.953125* proc.memory.percent‚17.254352569580078 cpu‚25.789511827018764 proc.cpu.threads‚17 network.recv‚195525% disk./.usageGB‚1173.7124061584473 disk.nvme0n1.in‚ 1852.4453125 disk.dm-0.in‚ 1982.2109375! disk.dm-1.out‚784.65673828125]ŠN‘Ђ(:† ßìòҼܴ¥ gpu.0.gpu‚90.0 gpu.0.memory‚18, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚50.0 gpu.0.powerWatts‚221.28' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚73.76 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0§±æ0Ñ+ _wandbruntime‚580  _runtime‚580Šè¸ÆÒÀ _step‚66 ' lr-AdamW/pg1‚0.00013906236980568487 ' lr-AdamW/pg2‚0.00013906236980568487 trainer/global_step‚849 % _timestamp‚1.7844609065555727e+09 _runtime‚ 580.944000558*˜ÊÓÄ ' lr-AdamW/pg1‚0.00013906236980568487 ' lr-AdamW/pg2‚0.00013906236980568487 trainer/global_step‚849 % _timestamp‚1.7844609065555727e+09 _runtime‚ 580.944000558 _step‚66B‘ 0Ô+ _wandbruntime‚580  _runtime‚580hûàÕ²O _step‚67 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 700558041600 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659456 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚6447411 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚65889377187328 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚15282996 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚6732221 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚6447594 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚183 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚65810952810095 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300224 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 700558041600 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12302 YKDtB̸eviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚66589829530112 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12373 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚49369057300992 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 % _timestamp‚1.7844609065567303e+09 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12373 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚4515407 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚65810848984313 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12302 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚65889360147968 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10144 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚6732223 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417275 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚65889360147968 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300224 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10142 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 700469382144 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301876736 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 700469382144 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚66589935228928 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10144 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 721917856768 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚8550773 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚66589935228928 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105698816 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚65889377187328 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 698547707817 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚1932004 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 215165440 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚66589829530112 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301876736 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚6732223 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚8550773 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚1932183 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚179 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 121055744 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚4515411 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105698816 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚8540631 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚15272852 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659456 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786422 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 _runtime‚ 580.94514258HÂZ¼'Ö¶O D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚4515411 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105698816 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚8540631 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚15272852 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659456 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786422 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 _runtime‚ 580.94514258 _step‚67 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 700558041600 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659456 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚6447411 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚65889377187328 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚15282996 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚6732221 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚6447594 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚183 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚65810952810095 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300224 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 700558041600 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12302 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚66589829530112 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12373 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚49369057300992 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 % _timestamp‚1.7844609065567303e+09 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12373 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚4515407 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚65810848984313 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12302 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚65889360147968 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10144 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚6732223 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417275 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚65889360147968 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300224 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10142 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 700469382144 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301876736 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 700469382144 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚66589935228928 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10144 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 721917856768 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚8550773 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚66589935228928 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105698816 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚65889377187328 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 698547707817 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚1932004 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 215165440 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚66589829530112 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301876736 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚6732223 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚8550773 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚1932183 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚179 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 121055744 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 IDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884171 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚15300459 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚1934327 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚65186591801888 _runtime‚ 581.369178614 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚6739850 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 701343764480 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚6454620 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚50147803595264 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 ODeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10158 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚15290301 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚8550451 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 701255097856 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12373 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚15300459 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚65964509814272 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372523008 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825894 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301876736 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105705984 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚4520467 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12373 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚8550451 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 701255097856 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚50148018753536 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚66665870618112 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 722608566272 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 _step‚68 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚65886026830199 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚6739852 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚66665870618112 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚49425289138688 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301876736 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚65964526853632 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚65885923004305 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 699417988951 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88666624 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10156 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1934157 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 % _timestamp‚1.7844609069806201e+09 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10156 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚6739850 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 722729614848 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0?gdvÙÁM F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301876736 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚65964526853632 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚65885923004305 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 699417988951 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88666624 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10156 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚1934157 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 % _timestamp‚1.7844609069806201e+09 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10156 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚6739850 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 722729614848 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 121048576 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884171 A9DeviceStatsMonitor.on_train_batch_end/segment.all.curHÀ†TQrent‚812 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚15300459 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚1934327 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚65186591801888 _runtime‚ 581.369178614 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚6739850 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 701343764480 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚6454620 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚50147803595264 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 ODeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10158 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚15290301 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚8550451 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 701255097856 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12373 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚15300459 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚65964509814272 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372523008 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825894 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301876736 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105705984 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚4520467 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12373 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚8550451 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 701255097856 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚50148018753536 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚66665870618112 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 722608566272 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 _step‚68 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚65886026830199 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚6739852 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚66665870618112 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚49425289138688 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200DlK¢ò0Ú+ _wandbruntime‚581  _runtime‚581X]ºÛû _runtime‚ 581.369889065 .loss/FBt2/detection_loss‚9.388316926849075 /loss/FRt2/assignment_loss‚2.406791855120405 2loss/SRqqt2/assignment_loss‚1.3344556803922707 1loss/FBt1/assignment_loss‚0.18019914528548586 1loss/FBt2/assignment_loss‚0.17959937059249173 1loss/SRqqt2/detection_loss‚0.8381253883396937 _step‚69 trainer/global_step‚849 &loss/total_loss‚27.311345870878576 0loss/FRt1/assignment_loss‚3.9923912350753423 2loss/SRqqt1/assignment_loss‚1.5764851604456678 0loss/SRqqt1/detection_loss‚2.392299425681195 0loss/FRt1/detection_loss‚0.47811348018693256 .loss/FRt2/detection_loss‚1.449639400976576 .loss/FBt1/detection_loss‚3.094928801933437 epoch‚2 % _timestamp‚1.7844609069817584e+09híÜÿ 0loss/FRt1/detection_loss‚0.47811348018693256 .loss/FRt2/detection_loss‚1.449639400976576 .loss/FBt1/detection_loss‚3.094928801933437 epoch‚2 % _timestamp‚1.7844609069817584e+09 _runtime‚ 581.369889065 .loss/FBt2/detection_loss‚9.388316926849075 /loss/FRt2/assignment_loss‚2.406791855120405 2loss/SRqqt2/assignment_loss‚1.3344556803922707 1loss/FBt1/assignment_loss‚0.18019914528548586 1loss/FBt2/assignment_loss‚0.17959937059249173 1loss/SRqqt2/detection_loss‚0.8381253883396937 _step‚69 trainer/global_step‚849 &loss/total_loss‚27.311345870878576 0loss/FRt1/assignment_loss‚3.9923912350753423 2loss/SRqqt1/assignment_loss‚1.5764851604456678 0loss/SRqqt1/detection_loss‚2.392299425681195E‰F rÝ‚(:ç îìòÒÙÉ¥) disk./.usagePercent‚76.45089889151171( disk.nvme1c1n1.out‚10842.79443359375 disk.md0.in‚0 disk.md0.out‚ 16.39453125* proc.memory.percent‚17.254352569580078 proc.cpu.threads‚17 network.recv‚198964& disk.nvme1n1.out‚10842.79443359375 disk.dm-0.out‚ 24564.5390625 disk.rbd0.in‚0 cpu‚25.784752331743217! disk.dm-1.out‚784.70361328125% disk./.usageGB‚1173.7124252319336 disk.nvme1c1n1.in‚ 129.765625 disk.dm-0.in‚ 1982.2109375 disk.dm-1.in‚0" disk.nvme0c0n1.in‚ 1852.4453125' disk.nvme0c0n1.out‚14541.4970703125 disk.nvme0n1.in‚ 1852.4453125 network.sent‚ 1.151705e+06 disk.nvme1n1.in‚ 129.765625 disk.rbd0.out‚0% disk.nvme0n1.out‚14541.4970703125$ memory_percent‚99.21314716339111( proc.memory.availableMB‚ 128.91796875! proc.memory.rssMB‚ 2826.953125â(:êžÞ‚(:“ îìòÒ•Ó³¥ gpu.0.gpu‚84.0 gpu.0.memory‚20, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚48.0 gpu.0.powerWatts‚114.427' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚38.14233333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0‚"½¢pß‚(:å ýìòÒåæ¤ disk.dm-0.in‚ 1982.2109375$ memory_percent‚99.21469688415527' proc.memory.availableMB‚ 128.6640625* proc.memory.percent‚17.254352569580078 cpu‚25.784736063563233 proc.cpu.threads‚17% disk.nvme1n1.out‚10843.7431640625! disk.dm-1.out‚784.96142578125$ disk./.usageGB‚1172.515438079834 disk.rbd0.in‚0" disk.nvme0c0n1.in‚ 1852.4453125 disk.nvme1c1n1.in‚ 129.765625' disk.nvme1c1n1.out‚10843.7431640625 disk.nvme1n1.in‚ 129.765625 disk.md0.out‚ 16.7421875! proc.memory.rssMB‚ 2826.953125 network.sent‚ 1.200238e+06 network.recv‚204511) disk./.usagePercent‚76.37293197067801 disk.nvme0n1.in‚ 1852.4453125& disk.nvme0n1.out‚14570.91845703125 disk.md0.in‚0 disk.dm-0.out‚24593.90234375 disk.dm-1.in‚0 disk.rbd0.out‚0( disk.nvme0c0n1.out‚14570.91845703125»Øåà‚(:’ ýìòÒˆ·Ù¤ gpu.0.gpu‚49.0 gpu.0.memory‚8, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚50.0 gpu.0.powerWatts‚214.865' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚71.62166666666667 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Œ÷ñ0á+ _wandbruntime‚613  _runtime‚613iÑœÅÆâÀ % _timestamp‚1.7844609394011474e+09 _runtime‚ 613.789547897 _step‚70 ' lr-AdamW/pg1‚0.00013902001926136925 ' lr-AdamW/pg2‚0.00013902001926136925 trainer/global_step‚899 Ù3ÊãÄ ' lr-AdamW/pg1‚0.00013902001926136925 ' lr-AdamW/pg2‚0.00013902001926136925 trainer/global_step‚899 % _timestamp‚1.7844609394011474e+09 _runtime‚ 613.789547897 _step‚70F<× 0ä+ _wandbruntime‚613  _runtime‚613~"sê¹'å³O VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 739819322368 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10142 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659456 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2041088 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 125250048 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301876736 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚69646860502528 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚16157298 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚70386662785536 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300224 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 219359744 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417275 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚6809295 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301876736 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚16147154 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12373 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10144 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚9043625 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚9033483 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659456 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 737786238625 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 739907981824 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12373 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚2041259 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚69564717610503 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884171 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚68826844585456 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚68826827546096 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12302 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12302 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚70386768484352 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚70386662785536 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚7113671 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚69646843463168 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚69646860502528 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825782 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300224 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚7113671 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚52938401464832 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚9033483 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 _runtime‚ 613.790424228 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚70386768484352 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105698816 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 762457763840 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 737873025047 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚9043625 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚171 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 739907981824 % _timestamp‚1.7844609394020693e+09 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚7113673 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372523008 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786422 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚6809470 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0æ:ü8½'æ·O VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659456 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2041088 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 125250048 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301876736 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚69646860502528 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚16157298 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚70386662785536 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300224 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 219359744 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417275 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚6809295 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301876736 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚16147154 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12373 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10144 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚9043625 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚9033483 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659456 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 737786238625 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 739907981824 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12373 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚2041259 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚69564717610503 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884171 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚68826844585456 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚68826827546096 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12302 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12302 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚70386768484352 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚70386662785536 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚7113671 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚69646843463168 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚69646860502528 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825782 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300224 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚7113671 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚52938401464832 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚9033483 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 _runtime‚ 613.790424228 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚70386768484352 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105698816 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 762457763840 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 737873025047 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚9043625 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚171 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 739907981824 % _timestamp‚1.7844609394020693e+09 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚7113673 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372523008 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786422 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚6809470 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 739819322368 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10142G{ÔÌe0ç+ _wandbruntime‚614  _runtime‚614 ]ŒVèºM QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚7121302 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 738656519759 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚68901118071488 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105705984 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88666624 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚9043303 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚69722010168832 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚69721993129472 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 114757120 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 trainer/global_step‚899 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚69721993129472 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301876736 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12302 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚6816714 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 _step‚72 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105705984 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 740693704704 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2043447 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚9043303 ODeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300224 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚16164603 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚52995561725440 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚52232081428992 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 740605038080 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚70462703873536 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚4773267 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884171 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚16174761 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚6816536 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚4773263 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0ûÝÄ&é¾M MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚7121302 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 740693704704 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚52232175538688 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786534 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 _runtime‚ 614.198052936 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚69639791630607 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 " _timestamp‚1.7844609398095e+09 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚69639687804713 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚7121302 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 738656519759 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚68901118071488 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105705984 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88666624 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚9043303 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚69722010168832 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚69721993129472 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 114757120 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 trainer/global_step‚899 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚69721993129472 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301876736 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12302 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚6816714 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 _step‚72 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105705984 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 740693704704 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2043447 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 I=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚9043303 ODeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300224 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚16164603 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚52995561725440 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚52232081428992 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 740605038080 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚70462703873536 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚4773267 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884171 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚16174761 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚6816536 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚4773263 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚73404343817728 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786422 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚5021007 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚73404326778368 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300224 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚187 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚183 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 803059739648 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884171 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 777024769433 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚7495123 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚73318378585129 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 _runtime‚ 645.921057594 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚9526335 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚9536477 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚54982924210688 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 _step‚75 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 779257922048 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10144 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚7170024 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 779257922048 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚9526335 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚74183601739776 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚74183601739776 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚2149013 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300224 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚7495121 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚73404343817728 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 777111555855 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10142 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚54982830100992 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301876736 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚55786109200384 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105698816 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825782 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 779169262592 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚55785889840640 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659456 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301876736 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚7169837 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 $ _timestamp‚1.784460971532705e+09 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚9536477 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚17031600 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10144 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚72541370855056 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚74183496040960 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 219359744 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚73404326778368 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167Ƶö†552 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12302 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 trainer/global_step‚949 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12373 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚5021011 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 779169262592 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12302 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 125250048 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚73318482410911êJù¼'ø¶O WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786422 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚5021007 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚73404326778368 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300224 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚187 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚183 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 803059739648 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884171 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 777024769433 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚7495123 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚73318378585129 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 _runtime‚ 645.921057594 K?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚9526335 SGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚9536477 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚54982924210688 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 _step‚75 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 779257922048 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10144 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚7170024 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 779257922048 OCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚9526335 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚74183601739776 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚74183601739776 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚2149013 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300224 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚7495121 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚73404343817728 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 777111555855 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10142 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚54982830100992 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301876736 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚55786109200384 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105698816 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825782 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 779169262592 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚55785889840640 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659456 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301876736 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚7169837 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 $ _timestamp‚1.784460971532705e+09 OCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚9536477 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚17031600 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10144 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚72541370855056 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚74183496040960 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 219359744 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚73404326778368 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12302 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 trainer/global_step‚949 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12373 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚5021011 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 779169262592 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12302 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 125250048 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚73318482410911 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚17021456 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚7495123 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417275 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚74183496040960 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚7495121 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚73404343817728 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2K\TãÙ0ù+ _wandbruntime‚646  _runtime‚646YêéüÃ&ú½M MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚7502752 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88666624 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12302 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚190 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 803995502592 _runtime‚ 646.327326047 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417275 MADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚9546311 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105705984 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 780043644928 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚17038905 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301876736 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚7177004 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚55842833894400 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚73393452605121 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884171 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786534 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10156 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 803866065408 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2150941 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚186 % _timestamp‚1.7844609719387603e+09 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105705984 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301876736 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚7502752 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚55038967828992 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚72615644341088 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12373 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚55039061938688 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 777808264033 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚73479476444672 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚74259537128960 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚9536155 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 223546880 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2151127 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 777895050567 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚17049063 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372523008 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚7502750 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚73479493484032 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 trainer/global_step‚949 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 779954978304 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚17049063 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10158 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300224 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚5026067 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚55843057441280 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10156 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 779954978304 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚7502750 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚5026063 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301876736 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚7502752 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚55038967828992 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚72615644341088 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12373 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚55039061938688 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 777808264033 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚73479476444672 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚74259537128960 MADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚9536155 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 223546880 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2151127 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 777895050567 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚17049063 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372523008 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚7502750 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚73479493484032 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 trainer/global_step‚949 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 779954978304 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚17049063 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10158 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300224 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚5026067 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚55843057441280 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10156 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 779954978304 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚7502750 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚5026063 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105705984 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 780043644928 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚17038905 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301876736 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚7177004 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚55842833894400 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚73393452605121 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884171 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786534 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10156 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 803866065408 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2150941 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚186 % _timestamp‚1.7844609719387603e+09 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105705984 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0LìtÎ0ü+ _wandbruntime‚646  _runtime‚646Axwþýø epoch‚2 2loss/SRqqt2/assignment_loss‚1.2842042642741058 1loss/FBt1/assignment_loss‚0.27890543006803886 /loss/FRt2/detection_loss‚1.3936662871833432 .loss/FBt1/detection_loss‚5.335559987517382 % _timestamp‚1.7844609719397001e+09 _step‚77 1loss/SRqqt2/detection_loss‚0.8085215513876158 &loss/total_loss‚22.882788971170235 trainer/global_step‚949 .loss/FBt2/detection_loss‚2.314011165030558 /loss/FRt1/assignment_loss‚3.918517175789744 .loss/FRt2/assignment_loss‚2.28497243642807 2loss/SRqqt1/assignment_loss‚1.6791459858655657 0loss/SRqqt1/detection_loss‚2.832358972112631 _runtime‚ 646.327810848 0loss/FBt2/assignment_loss‚0.2209964600646631 /loss/FRt1/detection_loss‚0.5319292554485211¸ñS»žþü 2loss/SRqqt1/assignment_loss‚1.6791459858655657 0loss/SRqqt1/detection_loss‚2.832358972112631 _runtime‚ 646.327810848 0loss/FBt2/assignment_loss‚0.2209964600646631 /loss/FRt1/detection_loss‚0.5319292554485211 epoch‚2 2loss/SRqqt2/assignment_loss‚1.2842042642741058 1loss/FBt1/assignment_loss‚0.27890543006803886 /loss/FRt2/detection_loss‚1.3936662871833432 .loss/FBt1/detection_loss‚5.335559987517382 % _timestamp‚1.7844609719397001e+09 _step‚77 1loss/SRqqt2/detection_loss‚0.8085215513876158 &loss/total_loss‚22.882788971170235 trainer/global_step‚949 .loss/FBt2/detection_loss‚2.314011165030558 æ²ç…d/loss/FRt1/assignment_loss‚3.918517175789744 .loss/FRt2/assignment_loss‚2.28497243642807M•ö–fÿ‚(:Û ¹íòÒŠ¯ñ¤ disk.md0.out‚ 18.1328125# disk.dm-0.out‚24658.80615234375$ memory_percent‚99.21455383300781$ proc.memory.availableMB‚128.6875" proc.memory.rssMB‚ 2826.7578125) proc.memory.percent‚17.25316047668457 cpu‚25.77086930150209 disk.rbd0.in‚0 disk.md0.in‚0$ disk.nvme1c1n1.out‚ 10846.9140625% disk./.usageGB‚1172.5714569091797 disk.dm-1.in‚0 disk.rbd0.out‚0" disk.nvme0c0n1.in‚ 1852.4453125 disk.nvme0n1.in‚ 1852.4453125$ disk.nvme0n1.out‚14636.201171875 disk.nvme1c1n1.in‚ 129.765625 proc.cpu.threads‚17 network.sent‚ 1.278289e+06 network.recv‚221373) disk./.usagePercent‚76.37658081154078 disk.nvme1n1.in‚ 129.765625" disk.nvme1n1.out‚ 10846.9140625 disk.dm-0.in‚ 1982.2109375! disk.dm-1.out‚785.50341796875& disk.nvme0c0n1.out‚14636.201171875ž Jž€‚(:“ ¹íòÒùÕ¥ gpu.0.gpu‚91.0 gpu.0.memory‚21, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚50.0 gpu.0.powerWatts‚222.298' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚74.09933333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0–±ÝWu‚(:ê ÈíòÒ‘ƾ¥ disk.nvme0n1.in‚ 1852.4453125& disk.nvme1n1.out‚10847.83740234375! disk.dm-1.out‚785.57373046875 disk.rbd0.in‚0 disk.rbd0.out‚0* proc.memory.percent‚17.258071899414062) disk./.usagePercent‚76.37659124744927& disk.nvme0n1.out‚14638.32958984375 disk.dm-1.in‚0( disk.nvme0c0n1.out‚14638.32958984375 proc.memory.rssMB‚ 2827.5625 cpu‚25.771093623356574% disk./.usageGB‚1172.5716171264648( disk.nvme1c1n1.out‚10847.83740234375 disk.md0.in‚0 disk.md0.out‚ 18.51171875# disk.dm-0.out‚24660.94873046875 proc.cpu.threads‚17 network.sent‚ 1.34239e+06 disk.nvme1c1n1.in‚ 129.765625 disk.nvme1n1.in‚ 129.765625 disk.dm-0.in‚ 1982.2109375" disk.nvme0c0n1.in‚ 1852.4453125$ memory_percent‚99.20985698699951( proc.memory.availableMB‚ 129.45703125 network.recv‚2255129<‚‚(:’ ÈíòÒ»ãù¤ gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚48.0 gpu.0.powerWatts‚114.607' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚38.202333333333335 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0\ôK™0ƒ+ _wandbruntime‚678  _runtime‚678b¨‡·Â„¼ % lr-AdamW/pg1‚0.000138921949128049 % lr-AdamW/pg2‚0.000138921949128049 trainer/global_step‚999 % _timestamp‚1.7844610038146136e+09 _runtime‚ 678.203041937 _step‚78€ÐŸÔÆ…À % lr-AdamW/pg1‚0.000138921949128049 % lr-AdamW/pg2‚0.000138921949128049 trainer/global_step‚999 % _timestamp‚1.7844610038146136e+09 _runtime‚ 678.203041937 _step‚78NìÔ§Ž0†+ _wandbruntime‚678  _runtime‚678Úˆ)¼'‡¶O TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 $ _timestamp‚1.784461003815788e+09 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 816350086663 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚2256759 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 123152896 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301876736 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10142 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚77072143385537 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚7876573 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 818519202816 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚77980434995200 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚206 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301876736 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚7876571 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚77161810093568 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚10029329 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825782 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 _runtime‚ 678.204244549 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10142 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12302 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 _step‚79 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659456 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚7876571 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚77072247211319 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372523008 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚10029329 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300224 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12373 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚17905902 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12302 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 818607862272 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚7530364 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚76255880085296 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10144 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚7530570 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884171 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12373 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚77161810093568 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2256557 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚58633553303552 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚77980434995200 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚77161827132928 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 818519202816 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105698816 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 818607862272 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659456 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417275 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300224 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚7876573 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚57789716500992 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 trainer/global_step‚999 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10144 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 843619539968 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚10019187 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚77980329296384 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚17895758 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 217262592 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚77161827132928 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚202 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 816263300241 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105698816 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚77980329296384 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚76255897124656 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786422 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 843742692864 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12302 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 818607862272 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚7530364 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚76255880085296 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10144 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚7530570 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884171 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12373 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚77161810093568 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2256557 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚58633553303552 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚77980434995200 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚77161827132928 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 818519202816 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105698816 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 818607862272 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659456 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417275 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300224 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚7876573 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚57789716500992 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 trainer/global_step‚999 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10144 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 843619539968 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚10019187 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚77980329296384 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚17895758 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 217262592 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚77161827132928 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚202 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 816263300241 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105698816 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚77980329296384 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚76255897124656 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786422 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 843742692864 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚10029007 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚7537728 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 trainer/global_step‚999 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚190 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚78056370384384 _step‚80 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚7884202 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚194 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2258861 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 _runtime‚ 678.618391038 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12302 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884171 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105705984 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚10039163 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2258671 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 819393585152 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚17913207 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 819304918528 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚7884202 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 223546880 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚58690280487936 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚7884200 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚77236959759872 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10158 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 817046794841 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚17923365 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300224 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12373 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 ODeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301876736 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 % _timestamp‚1.7844610042298222e+09 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚58690504034816 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚77147217405529 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚76330170610688 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚17913207 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚78056370384384 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚77236959759872 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 819304918528 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300224 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 817133581375 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786534 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚10029007 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10156 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 129437184 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚77236976799232 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚7537534 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚17923365 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825894 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10156 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚78056264678400 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚57845854228992 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 844555696128 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0Ží®Ë&‹ÅM TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 819304918528 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚7884202 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 223546880 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚58690280487936 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚7884200 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚77236959759872 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10158 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 817046794841 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚17923365 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300224 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12373 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 ODeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301876736 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 % _timestamp‚1.7844610042298222e+09 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚58690504034816 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚77147217405529 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚76330170610688 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚17913207 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚78056370384384 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚77236959759872 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 819304918528 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300224 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 817133581375 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786534 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚10029007 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10156 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 129437184 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚77236976799232 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚7537534 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚17923365 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825894 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10156 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚78056264678400 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚57845854228992 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 844555696128 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚5278867 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚77147321231423 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚10029007 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚7537728 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 trainer/global_step‚999 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚190 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚78056370384384 _step‚80 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚7884202 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚194 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2258861 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 _runtime‚ 678.618391038 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12302 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884171 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105705984 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚10039163 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2258671 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 819393585152 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚17913207 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200PŒ•¡0Œ+ _wandbruntime‚678  _runtime‚678‰Ófáû /loss/FRt1/detection_loss‚0.5821091122554563 1loss/SRqqt2/detection_loss‚0.7688542895255817 _runtime‚ 678.618899889 &loss/total_loss‚22.642084761733365 /loss/FRt2/assignment_loss‚2.241768856561893 2loss/SRqqt1/assignment_loss‚1.6720569596121397 1loss/FBt2/assignment_loss‚0.11644878069989015 1loss/SRqqt1/detection_loss‚3.3954088037752803 .loss/FBt2/detection_loss‚3.818564450513734 $ _timestamp‚1.784461004230814e+09 _step‚81 /loss/FRt1/assignment_loss‚3.936852531138913 1loss/FBt1/assignment_loss‚0.15597674275169435 epoch‚2 trainer/global_step‚999 2loss/SRqqt2/assignment_loss‚1.1583316506007946 /loss/FRt2/detection_loss‚1.3062788268555101 /loss/FBt1/detection_loss‚3.4894337574424767z<“„Žÿ 1loss/SRqqt1/detection_loss‚3.3954088037752803 .loss/FBt2/detection_loss‚3.818564450513734 $ _timestamp‚1.784461004230814e+09 _step‚81 /loss/FRt1/assignment_loss‚3.936852531138913 1loss/FBt1/assignment_loss‚0.15597674275169435 epoch‚2 trainer/global_step‚999 2loss/SRqqt2/assignment_loss‚1.1583316506007946 /loss/FRt2/detection_loss‚1.3062788268555101 /loss/FBt1/detection_loss‚3.4894337574424767 /loss/FRt1/detection_loss‚0.5821091122554563 1loss/SRqqt2/detection_loss‚0.7688542895255817 _runtime‚ 678.618899889 &loss/total_loss‚22.642084761733365 /loss/FRt2/assignment_loss‚2.241768856561893 2loss/SRqqt1/assignment_loss‚1.6720569596121397 1loss/FBt2/assignment_loss‚0.11644878069989015QsFi‚(:Þ ×íòÒÛÓƤ proc.memory.rssMB‚ 2827.5625* proc.memory.percent‚17.258071899414062) disk./.usagePercent‚76.37661708874649 disk.dm-1.out‚ 785.658203125 disk.rbd0.in‚0" disk.nvme0n1.out‚ 14639.0546875 cpu‚25.766806582397212 network.recv‚228998% disk./.usageGB‚1172.5720138549805$ disk.nvme0c0n1.out‚ 14639.0546875 disk.nvme0n1.in‚ 1852.4453125( disk.nvme1c1n1.out‚10848.22119140625 disk.md0.in‚0& proc.memory.availableMB‚ 129.609375 proc.cpu.threads‚17 network.sent‚ 1.345354e+06 disk.rbd0.out‚0 disk.nvme1n1.in‚ 129.765625& disk.nvme1n1.out‚10848.22119140625 disk.md0.out‚ 18.7265625 disk.dm-0.in‚ 1982.2109375# disk.dm-0.out‚24661.49951171875 disk.dm-1.in‚0" disk.nvme0c0n1.in‚ 1852.4453125 disk.nvme1c1n1.in‚ 129.765625$ memory_percent‚99.20892715454102ßWhž‚(:“ ×íòÒ³´ª¥ gpu.0.gpu‚90.0 gpu.0.memory‚16, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚50.0 gpu.0.powerWatts‚214.083' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚71.36099999999999 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0V)X^n‘‚(:ã æíòÒ±ÒÕ¤ disk.nvme1n1.in‚ 129.765625$ memory_percent‚99.20849800109863 disk.nvme0n1.in‚ 1852.4453125 network.sent‚ 1.407718e+06( disk./.usagePercent‚76.3766153494284% disk./.usageGB‚1172.5719871520996# disk.dm-0.out‚24664.07373046875 disk.dm-1.in‚0 disk.rbd0.out‚0 disk.md0.in‚0 network.recv‚233564 disk.dm-0.in‚ 1982.2109375( disk.nvme0c0n1.out‚14641.52783203125& disk.nvme0n1.out‚14641.52783203125 disk.md0.out‚ 19.00390625' proc.memory.availableMB‚ 129.6796875 proc.memory.rssMB‚ 2827.5625* proc.memory.percent‚17.258071899414062" disk.nvme0c0n1.in‚ 1852.4453125 disk.dm-1.out‚ 785.705078125 disk.rbd0.in‚0 disk.nvme1c1n1.in‚ 129.765625% disk.nvme1n1.out‚10848.9794921875 cpu‚25.7629670526694 proc.cpu.threads‚17' disk.nvme1c1n1.out‚10848.9794921875ØLeo“’‚(:ˆ æíòÒ㜥 gpu.0.gpu‚91.0 gpu.0.memory‚19, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚49.0 gpu.0.powerWatts‚114.981' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚38.327 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Ê«S0“+ _wandbruntime‚711  _runtime‚711°[ãÇ”Á ' lr-AdamW/pg1‚0.00013886623583790579 ' lr-AdamW/pg2‚0.00013886623583790579 trainer/global_step‚1049 % _timestamp‚1.7844610367749915e+09 _runtime‚ 711.163978567 _step‚82!ëNË•Å trainer/global_step‚1049 % _timestamp‚1.7844610367749915e+09 _runtime‚ 711.163978567 _step‚82 ' lr-AdamW/pg1‚0.00013886623583790579 ' lr-AdamW/pg2‚0.00013886623583790579Rz‹|D0–+ _wandbruntime‚711  _runtime‚711ÂÞ¥¾'—¸O QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚171 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 857957802496 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10144 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825782 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 857957802496 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚61480976041472 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚10522181 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301876736 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚80825908185945 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚18780204 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 % _timestamp‚1.7844610367757478e+09 _runtime‚ 711.164276958 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚81777162551808 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300224 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚79970406354896 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786422 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚8258021 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10142 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301876736 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚5526607 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2364401 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚8258023 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚79970423394256 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚80919293408768 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659456 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659456 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚80826012011727 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚8258023 trainer/global_step‚1049 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10142 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚61480762973184 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 884160072192 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 _step‚83 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚175 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 884279030784 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 857869143040 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300224 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚81777268250624 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372523008 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12302 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚2364572 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 857869143040 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚60596602900992 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚18770060 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚80919310448128 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚60596697010688 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚8258021 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 118958592 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417275 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚10522181 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10144 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12302 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚81777268250624 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚80919293408768 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚7891008 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 855588617471 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12373 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚10512039 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 213068288 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚80919310448128 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚81777162551808 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350¿å®˜¼O H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12302 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚81777268250624 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚80919293408768 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚7891008 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 855588617471 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12373 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚10512039 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 213068288 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚80919310448128 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚81777162551808 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚171 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 857957802496 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10144 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825782 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 857957802496 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚61480976041472 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚10522181 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301876736 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚80825908185945 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚18780204 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 % _timestamp‚1.7844610367757478e+09 _runtime‚ 711.164276958 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚81777162551808 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300224 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚79970406354896 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786422 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚8258021 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10142 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301876736 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚5526607 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2364401 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚8258023 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚79970423394256 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚80919293408768 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659456 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659456 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚80826012011727 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚8258023 trainer/global_step‚1049 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10142 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚61480762973184 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 884160072192 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 _step‚83 JBDeviceSta7P@ tsMonitor.on_train_batch_start/inactive_split.all.current‚175 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 884279030784 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 857869143040 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300224 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚81777268250624 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372523008 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12302 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚2364572 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 857869143040 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚60596602900992 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚18770060 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚80919310448128 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚60596697010688 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚8258021 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 118958592 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417275 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚10522181 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10144 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0SªêH|0™+ _wandbruntime‚711  _runtime‚711¡¾È&šÂM MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚8265650 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚61537924413440 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301876736 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚60652740628992 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10156 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚10521859 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 858654858752 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚80994443075072 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12302 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 125242880 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚8265652 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300224 _runtime‚ 711.581776643 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825894 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚80044713919648 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚18787509 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12373 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚10532015 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚10521859 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2366535 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚80994443075072 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚81853097933824 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚5531663 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚18797667 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105705984 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚8265652 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚7898198 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372523008 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 trainer/global_step‚1049 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417275 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚81853203639808 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 884964431872 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 858743525376 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88666624 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12373 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 885089674752 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884171 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚80994460114432 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚61537705060864 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚80901086031831 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚80900982205937 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2366720 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚80994460114432 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚7898387 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 219352576 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 858743525376 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚189 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚185 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105705984 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300224 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚18797667 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚80044696880288 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786534 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚81853203639808 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 858654858752 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 % _timestamp‚1.7844610371931837e+09 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88666624ÈxüÌ&›ÆM SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417275 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚81853203639808 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 884964431872 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 858743525376 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88666624 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12373 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 885089674752 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884171 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚80994460114432 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚61537705060864 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚80901086031831 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚80900982205937 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2366720 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚80994460114432 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚7898387 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 219352576 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 XGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 858743525376 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚189 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚185 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105705984 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300224 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚18797667 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚80044696880288 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786534 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚81853203639808 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 858654858752 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 % _timestamp‚1.7844610371931837e+09 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88666624 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚8265650 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚61537924413440 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301876736 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚60652740628992 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10156 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚10521859 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 858654858752 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚80994443075072 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12302 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 125242880 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚8265652 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300224 _runtime‚ 711.581776643 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825894 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚80044713919648 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚18787509 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12373 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚10532015 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚10521859 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2366535 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚80994443075072 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚81853097933824 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚5531663 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚18797667 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105705984 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚8265652 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚7898198 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372523008 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 trainer/global_step‚1049Tʤk0œ+ _wandbruntime‚711  _runtime‚711›?NÈþø 0loss/FRt2/assignment_loss‚2.0978642241477967 1loss/FBt1/assignment_loss‚0.21181307933266524 trainer/global_step‚1049 _runtime‚ 711.582337723 epoch‚2 0loss/SRqqt1/assignment_loss‚1.46161821872378 0loss/FBt2/assignment_loss‚0.2748270312747144 .loss/FRt2/detection_loss‚1.502419116122452 0loss/SRqqt1/detection_loss‚2.444910507046077 1loss/SRqqt2/detection_loss‚0.8389248310557168 .loss/FBt2/detection_loss‚4.950926710983963 /loss/FBt1/detection_loss‚2.5788786776395614 _step‚85 %loss/total_loss‚21.90395808156955 1loss/SRqqt2/assignment_loss‚1.175539506112989 /loss/FRt1/detection_loss‚0.5191962254176874 % _timestamp‚1.7844610371942575e+09 0loss/FRt1/assignment_loss‚3.8470399537121462f|õ°žü 1loss/SRqqt2/detection_loss‚0.8389248310557168 .loss/FBt2/detection_loss‚4.950926710983963 /loss/FBt1/detection_loss‚2.5788786776395614 _step‚85 %loss/total_loss‚21.90395808156955 1loss/SRqqt2/assignment_loss‚1.175539506112989 /loss/FRt1/detection_loss‚0.5191962254176874 % _timestamp‚1.7844610371942575e+09 0loss/FRt1/assignment_loss‚3.8470399537121462 0loss/FRt2/assignment_loss‚2.0978642241477967 1loss/FBt1/assignment_loss‚0.21181307933266524 trainer/global_step‚1049 _runtime‚ 711.582337723 epoch‚2 0loss/SRqqt1/assignment_loss‚1.46161821872378 0loss/FBt2/assignment_loss‚0.2748270312747144 .loss/FRt2/detection_loss‚1.502419116122452 0loss/SRqqt1/detection_loss‚2.444910507046077UJ¡RteŸ‚(:Ú õíòÒäî÷¤) disk./.usagePercent‚76.37660665283799% disk./.usageGB‚1172.5718536376953 disk.rbd0.out‚0& disk.nvme1c1n1.out‚10849.392578125 disk.nvme1n1.in‚ 129.765625 disk.dm-0.in‚ 1982.2109375# disk.dm-0.out‚24664.71044921875# proc.memory.availableMB‚129.625 network.sent‚ 1.426614e+06 disk.rbd0.in‚0$ disk.nvme1n1.out‚10849.392578125 disk.md0.in‚0 proc.memory.rssMB‚ 2827.5625* proc.memory.percent‚17.258071899414062 cpu‚25.75961440519133 disk.dm-1.in‚0" disk.nvme0c0n1.in‚ 1852.4453125 disk.nvme0n1.in‚ 1852.4453125 disk.nvme1c1n1.in‚ 129.765625$ memory_percent‚99.20883178710938 proc.cpu.threads‚17 network.recv‚237025 disk.dm-1.out‚785.8125( disk.nvme0c0n1.out‚14642.30615234375& disk.nvme0n1.out‚14642.30615234375 disk.md0.out‚ 19.19921875¾ŽÏPž ‚(:“ õíòÒÀô©¥ gpu.0.gpu‚91.0 gpu.0.memory‚17, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚50.0 gpu.0.powerWatts‚225.255' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚75.08500000000001 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0-EÏØl¡‚(:á „îòÒãÜפ" disk.nvme0c0n1.in‚ 1852.4453125( disk.nvme0c0n1.out‚14644.91162109375$ memory_percent‚99.60002899169922$ proc.memory.availableMB‚65.53125 proc.memory.rssMB‚ 2827.5625 network.recv‚241001% disk.nvme1n1.out‚10850.2119140625& disk.nvme0n1.out‚14644.91162109375 disk.nvme1c1n1.in‚ 129.765625' disk.nvme1c1n1.out‚10850.2119140625* proc.memory.percent‚17.258071899414062 proc.cpu.threads‚17% disk./.usageGB‚1172.5723838806152 disk.nvme1n1.in‚ 129.765625 disk.rbd0.in‚0 disk.nvme0n1.in‚ 1852.4453125) disk./.usagePercent‚76.37664119072562 disk.md0.out‚ 19.6484375 disk.dm-0.in‚ 1982.2109375# disk.dm-0.out‚24667.09423828125 disk.dm-1.in‚0 disk.rbd0.out‚0 cpu‚25.756433832146513 network.sent‚ 1.474862e+06 disk.md0.in‚0 disk.dm-1.out‚ 785.88671875!öW‚“¢‚(:ˆ „îòÒç查 gpu.0.gpu‚91.0 gpu.0.memory‚20, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚50.0 gpu.0.powerWatts‚117.813' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚39.271 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0r¹ü0£+ _wandbruntime‚743  _runtime‚743Œqñ^ǤÁ _step‚86 ' lr-AdamW/pg1‚0.00013880607507462696 ' lr-AdamW/pg2‚0.00013880607507462696 trainer/global_step‚1099 % _timestamp‚1.7844610691886158e+09 _runtime‚ 743.576989265J‹—é˥Š' lr-AdamW/pg2‚0.00013880607507462696 trainer/global_step‚1099 % _timestamp‚1.7844610691886158e+09 _runtime‚ 743.576989265 _step‚86 ' lr-AdamW/pg1‚0.00013880607507462696VÂ3Uë0¦+ _wandbruntime‚743  _runtime‚743OB(­”§¸O UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚85574101506048 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 % _timestamp‚1.7844610691896164e+09 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚8639473 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 924755067392 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 127347200 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚84676776723968 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12302 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚85573995807232 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚19644362 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300224 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚11015033 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚11004891 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚63403583410688 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10144 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 897307742720 WGDeviceStatsMonitor.on_train_batch_start>>]Ð*/allocated_bytes.large_pool.peak‚ 12065026048 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚8250922 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚185 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚83684949663856 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚84676776723968 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚8639471 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚181 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825782 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚84579672986353 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417275 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 924627720192 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786422 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚8639473 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372523008 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300224 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚19654506 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12373 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12302 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚64328117021184 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10144 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105698816 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚63403489300992 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 894740361857 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 897219083264 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚84579776812135 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884171 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚8639471 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659456 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 % _timestamp‚1.7844610691896164e+09 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚8639473 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 924755067392 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 127347200 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚84676776723968 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12302 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚85573995807232 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚19644362 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300224 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚11015033 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚11004891 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚63403583410688 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10144 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 897307742720 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚8250922 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚185 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚83684949663856 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚84676776723968 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚8639471 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚181 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825782 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚84579672986353 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417275 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 ^MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 924627720192 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786422 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚8639473 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372523008 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300224 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚19654506 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12373 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12302 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚64328117021184 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10144 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105698816 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚63403489300992 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 894740361857 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 897219083264 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚84579776812135 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884171 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚8639471 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659456 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760WRaÓ0©+ _wandbruntime‚743  _runtime‚743ùŠÚeÈ&ªÂM RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 _runtime‚ 743.989594112 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚11014711 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚19671969 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12302 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚8257981 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105705984 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 898004798976 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚83759240189248 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300224 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300224 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88666624 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 227741184 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚85650036895232 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚84751926390272 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88666624 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚84654747006345 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚11024867 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12373 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚63459627028992 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884171 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301876736 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2473514 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 925564924928 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 895523856457 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10158 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105705984 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 trainer/global_step‚1099 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚84751943429632 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚8257810 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 925431293440 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825894 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚64385286063616 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786534Õ_DŒù«ÆM =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300224 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300224 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88666624 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 227741184 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚85650036895232 VCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚84751926390272 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88666624 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚84654747006345 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚11024867 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12373 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚63459627028992 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884171 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301876736 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2473514 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 925564924928 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 895523856457 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10158 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105705984 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 trainer/global_step‚1099 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚84751943429632 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚8257810 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 925431293440 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825894 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚64385286063616 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786534 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 _runtime‚ 743.989594112 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚11014711 ODeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚19671969 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12302 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚8257981 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105705984 WFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 898004798976 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚83759240189248 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760X¢rÄ0¬+ _wandbruntime‚743  _runtime‚743Òô£K­ü trainer/global_step‚1099 % _timestamp‚1.7844610696020188e+09 2loss/SRqqt1/assignment_loss‚1.4094195727713223 0loss/SRqqt1/detection_loss‚2.516156148127057 0loss/FRt1/assignment_loss‚3.7509537386226897 2loss/SRqqt2/assignment_loss‚1.1382980275019272 _runtime‚ 743.990102112 1loss/FBt1/assignment_loss‚0.20964572104982213 1loss/FBt2/assignment_loss‚0.28739297019309545 /loss/FRt1/detection_loss‚0.5246058385244156 .loss/FBt1/detection_loss‚2.875967244949787 %loss/total_loss‚21.64771838915157 epoch‚2 _step‚89 0loss/FRt2/assignment_loss‚2.1240913183567764 /loss/FRt2/detection_loss‚1.3809714265381698 1loss/SRqqt2/detection_loss‚0.8416854085982063 .loss/FBt2/detection_loss‚4.588530973918302æn s®€ %loss/total_loss‚21.64771838915157 epoch‚2 _step‚89 0loss/FRt2/assignment_loss‚2.1240913183567764 /loss/FRt2/detection_loss‚1.3809714265381698 1loss/SRqqt2/detection_loss‚0.8416854085982063 .loss/FBt2/detection_loss‚4.588530973918302 trainer/global_step‚1099 % _timestamp‚1.7844610696020188e+09 2loss/SRqqt1/assignment_loss‚1.4094195727713223 0loss/SRqqt1/detection_loss‚2.516156148127057 0loss/FRt1/assignment_loss‚3.7509537386226897 2loss/SRqqt2/assignment_loss‚1.1382980275019272 _runtime‚ 743.990102112 1loss/FBt1/assignment_loss‚0.20964572104982213 1loss/FBt2/assignment_loss‚0.28739297019309545 /loss/FRt1/detection_loss‚0.5246058385244156 .loss/FBt1/detection_loss‚2.875967244949787Y…¿þh¯‚(:Ý “îòÒÙ¥* proc.memory.percent‚17.258071899414062 network.sent‚ 1.49365e+06' disk.nvme1c1n1.out‚10940.1494140625 disk.nvme1n1.in‚ 129.765625 disk.rbd0.out‚0& disk.nvme0n1.out‚14978.25537109375 cpu‚25.75268241397446 proc.cpu.threads‚17 disk.nvme1c1n1.in‚ 129.765625% disk.nvme1n1.out‚10940.1494140625 disk.dm-0.in‚ 1982.2109375 disk.dm-1.out‚ 786.52734375 disk.rbd0.in‚0 proc.memory.rssMB‚ 2827.5625% disk./.usageGB‚1173.3286209106445 disk.md0.in‚0 disk.md0.out‚ 20.0390625" disk.nvme0c0n1.in‚ 1852.4453125# memory_percent‚99.6025562286377 network.recv‚244364& disk./.usagePercent‚76.42589942424# disk.dm-0.out‚25088.89111328125 disk.dm-1.in‚0( disk.nvme0c0n1.out‚14978.25537109375 disk.nvme0n1.in‚ 1852.4453125& proc.memory.availableMB‚ 65.1171875±Ñ3­ž°‚(:“ “îòÒ²‘¼¥ gpu.0.gpu‚91.0 gpu.0.memory‚21, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚49.0 gpu.0.powerWatts‚115.034' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚38.34466666666667 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ý‘i±‚(:Þ ¢îòÒþȤ) disk./.usagePercent‚76.42590762388238% disk./.usageGB‚1173.3287467956543 disk.nvme0n1.in‚ 1852.4453125 disk.md0.in‚0 disk.md0.out‚ 20.4296875 network.sent‚ 1.541949e+06% disk.nvme0n1.out‚15330.9228515625' disk.nvme1c1n1.out‚10940.9716796875 disk.nvme1n1.in‚ 129.765625% disk.nvme1n1.out‚10940.9716796875! disk.dm-0.out‚25441.392578125 disk.dm-1.out‚ 786.69140625* proc.memory.percent‚17.258071899414062' disk.nvme0c0n1.out‚15330.9228515625# memory_percent‚99.6025800704956 proc.memory.rssMB‚ 2827.5625 cpu‚25.74977118482917 proc.cpu.threads‚17 network.recv‚248378" disk.nvme0c0n1.in‚ 1852.4453125 disk.nvme1c1n1.in‚ 129.765625 disk.dm-0.in‚ 1982.2109375 disk.dm-1.in‚0 disk.rbd0.in‚0 disk.rbd0.out‚0' proc.memory.availableMB‚ 65.11328125tÕ¥ ‘²‚(:† ¢îòÒä„•¥ gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚48.0 gpu.0.powerWatts‚114.555' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚38.185 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0H §-0³+ _wandbruntime‚775  _runtime‚7753êW†Æ´À ' lr-AdamW/pg1‚0.00013874147070222595 ' lr-AdamW/pg2‚0.00013874147070222595 trainer/global_step‚1149 % _timestamp‚1.7844611014683871e+09 _runtime‚ 775.85679199 _step‚90›îÕ%浀 ' lr-AdamW/pg1‚0.00013874147070222595 ' lr-AdamW/pg2‚0.00013874147070222595 trainer/global_step‚1149 % _timestamp‚1.7844611014683871e+09 _runtime‚ 775.85679199 _step‚90Zø*K:0¶+ _wandbruntime‚775  _runtime‚775tÜ¥¥¾'·¸O TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2577174 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 131541504 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12373 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 936657682944 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚88434260039168 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚89370829062656 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300224 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417275 % _timestamp‚1.7844611014693909e+09 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300224 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12373 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 934065679087 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786422 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12302 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚8609381 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚66210375700992 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚89370829062656 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10144 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 936569023488 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚9020921 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚11507885 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚87399475933456 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301876736 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 936569023488 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚11507885 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚9020923 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372523008 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 933978892665 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚20518664 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 936657682944 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚187 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825782 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚88434277078528 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12302 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚6032207 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 225651200 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚88434277078528 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚8609568 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10144 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 965194714112 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659456 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚6032211 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚11497743 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812S/ÇvÂ'¸¼O PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚11497743 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2577174 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 131541504 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12373 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 936657682944 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚88434260039168 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚89370829062656 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300224 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417275 % _timestamp‚1.7844611014693909e+09 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300224 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12373 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 934065679087 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786422 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12302 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚8609381 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚66210375700992 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚89370829062656 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10144 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 936569023488 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚9020921 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚11507885 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚87399475933456 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301876736 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 936569023488 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚11507885 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚9020923 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372523008 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 933978892665 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚20518664 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 936657682944 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚187 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825782 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚88434277078528 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12302 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚6032207 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 225651200 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚88434277078528 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚8609568 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10144 bQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 965194714112 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659456 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚6032211 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0[;œ 80¹+ _wandbruntime‚776  _runtime‚776Ô¹ýüåºÁM [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 937443405824 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚11507563 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2579318 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚9028550 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚11507563 O6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚20536113 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372523008 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚66266607538688 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 129437184 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚8616581 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚6037263 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 937354739200 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚88509409705472 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚2054‘Æ™â6271 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚9028550 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚11517719 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 965869236224 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88666624 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12373 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚88408511806753 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚173 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825894 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417275 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 $ _timestamp‚1.784461101880179e+09 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 965998673408 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 934762387265 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 _step‚92 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10156 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884171 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚87473749419488 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10156 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300224 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚66266513428992 _runtime‚ 776.268880612 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105705984 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105705984 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301876736 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚88408615632647 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 223546880 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚11517719 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12302 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2579491 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚89446870150656 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚87473766458848 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚177 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚67232382665216 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚88509426744832 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚6037267 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786534 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0¨‚2Ë&»ÅM >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚177 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚67232382665216 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚88509426744832 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚6037267 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786534 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 937443405824 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚11507563 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2579318 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚9028550 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚11507563 O6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚20536113 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372523008 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚66266607538688 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 129437184 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚8616581 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚6037263 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 TCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 937354739200 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚88509409705472 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚20546271 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚9028550 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚11517719 \KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 965869236224 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88666624 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12373 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚88408511806753 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚173 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825894 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417275 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 $ _timestamp‚1.784461101880179e+09 `ODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 965998673408 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 934762387265 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 _step‚92 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10156 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884171 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚87473749419488 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10156 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300224 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚66266513428992 _runtime‚ 776.268880612 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105705984 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105705984 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301876736 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚88408615632647 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 223546880 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚11517719 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12302 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2579491 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚89446870150656 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚87473766458848\‹¼Ì/0¼+ _wandbruntime‚776  _runtime‚776 Ç½ü 2loss/SRqqt2/assignment_loss‚1.1696729893309685 /loss/FRt1/detection_loss‚0.5256755178159801 1loss/SRqqt2/detection_loss‚0.7421290999964569 1loss/SRqqt1/detection_loss‚2.3185500746989427 trainer/global_step‚1149 $ _timestamp‚1.784461101881268e+09 &loss/total_loss‚19.338781604214255 1loss/FBt1/assignment_loss‚0.15994311914074305 /loss/FRt2/detection_loss‚1.3420169442475418 /loss/FBt1/detection_loss‚2.8400318268234916 _step‚93 /loss/FRt1/assignment_loss‚3.723325901925564 1loss/FBt2/assignment_loss‚0.23049078365572268 /loss/FBt2/detection_loss‚2.8115146938184172 _runtime‚ 776.269429263 epoch‚2 /loss/FRt2/assignment_loss‚1.962736636725677 1loss/SRqqt1/assignment_loss‚1.512694016034747jYY¾€ epoch‚2 /loss/FRt2/assignment_loss‚1.962736636725677 1loss/SRqqt1/assignment_loss‚1.512694016034747 2loss/SRqqt2/assignment_loss‚1.1696729893309685 /loss/FRt1/detection_loss‚0.5256755178159801 1loss/SRqqt2/detection_loss‚0.7421290999964569 1loss/SRqqt1/detection_loss‚2.3185500746989427 trainer/global_step‚1149 $ _timestamp‚1.784461101881268e+09 &loss/total_loss‚19.338781604214255 1loss/FBt1/assignment_loss‚0.15994311914074305 /loss/FRt2/detection_loss‚1.3420169442475418 /loss/FBt1/detection_loss‚2.8400318268234916 _step‚93 /loss/FRt1/assignment_loss‚3.723325901925564 1loss/FBt2/assignment_loss‚0.23049078365572268 /loss/FBt2/detection_loss‚2.8115146938184172 _runtime‚ 776.269429263]UîTêo¿‚(:ä ±îòÒîæç¤% disk./.usageGB‚1173.3287200927734 disk.md0.in‚0* proc.memory.percent‚17.258071899414062 proc.cpu.threads‚17 network.sent‚ 1.560691e+06 disk.dm-1.in‚0 disk.rbd0.in‚0 disk.nvme0n1.in‚ 1852.4453125& disk.nvme0n1.out‚15334.33837890625 disk.md0.out‚ 20.640625 disk.dm-0.in‚ 1982.2109375 disk.dm-1.out‚786.8134765625 disk.rbd0.out‚0" disk.nvme0c0n1.in‚ 1852.4453125 disk.nvme1c1n1.in‚ 129.765625( disk.nvme1c1n1.out‚10941.73974609375& disk.nvme1n1.out‚10941.73974609375 network.recv‚251708! disk.dm-0.out‚25444.982421875( disk.nvme0c0n1.out‚15334.33837890625 disk.nvme1n1.in‚ 129.765625$ memory_percent‚99.60381984710693' proc.memory.availableMB‚ 64.91015625 proc.memory.rssMB‚ 2827.5625 cpu‚25.747553304997286( disk./.usagePercent‚76.4259058845643·3^š›À‚(: ±îòÒž¹¦¥ gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚50.0 gpu.0.powerWatts‚208.66' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚69.55333333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Ì¥*XžÁ‚(:“ ÀîòÒïú¼¤ gpu.0.gpu‚91.0 gpu.0.memory‚22, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚48.0 gpu.0.powerWatts‚114.839' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚38.27966666666667 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0“Œå7e‚(:Ú ÀîòÒ£È㤠disk.dm-0.in‚ 1982.2109375 disk.dm-1.in‚0 disk.dm-1.out‚786.8876953125" disk.nvme0c0n1.in‚ 1852.4453125 disk.md0.in‚0 network.sent‚ 1.61095e+06 disk.rbd0.in‚0& disk.nvme1c1n1.out‚10943.021484375$ proc.memory.availableMB‚64.40625* proc.memory.percent‚17.258071899414062 proc.cpu.threads‚17% disk./.usageGB‚1173.3286628723145" disk.dm-0.out‚25450.8173828125 disk.rbd0.out‚0& disk.nvme0c0n1.out‚15339.693359375 disk.nvme0n1.in‚ 1852.4453125 disk.nvme1c1n1.in‚ 129.765625 disk.nvme1n1.in‚ 129.765625$ disk.nvme1n1.out‚10943.021484375 network.recv‚256195) disk./.usagePercent‚76.42590215745413$ disk.nvme0n1.out‚15339.693359375 disk.md0.out‚ 20.984375$ memory_percent‚99.60689544677734 proc.memory.rssMB‚ 2827.5625 cpu‚25.744817758207535æA‰j0Ã+ _wandbruntime‚808  _runtime‚808çÄ ¢ÇÄÁ ' lr-AdamW/pg1‚0.00013867242687012085 ' lr-AdamW/pg2‚0.00013867242687012085 trainer/global_step‚1199 % _timestamp‚1.7844611337935388e+09 _runtime‚ 808.181851195 _step‚94©ÈW:ËÅÅ ' lr-AdamW/pg1‚0.00013867242687012085 ' lr-AdamW/pg2‚0.00013867242687012085 trainer/global_step‚1199 % _timestamp‚1.7844611337935388e+09 _runtime‚ 808.181851195 _step‚94^Vae}0Æ+ _wandbruntime‚808  _runtime‚808øæ€2À'ǺO E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚21392966 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚93167662318080 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚9402373 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12302 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚70023012837888 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚177 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ZIDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 976007623168 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301876736 ]LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 976007623168 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚9402371 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2682887 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚91113985163696 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚92087306412951 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚12000737 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10142 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1005531377152 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300224 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚6285011 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 219359744 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825782 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚181 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12302 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚70022793478144 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚92191743354368 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚6285007 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚69017262100992 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105698816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659456 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 975918963712 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚8968075 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 % _timestamp‚1.7844611337942135e+09 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10144 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚21403110 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786422 trainer/global_step‚1199 _runtime‚ 808.183735248 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884171 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 973217423473 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚93167768016896 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚92191760393728 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10144 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚11990595 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚92191760393728 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 _step‚95 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10142 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚69017356210688 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301876736 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105698816 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚2683064 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1005656627200 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚93167662318080 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659456 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 975918963712 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372523008 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚11990595 ]LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 973304209895 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12302 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚70022793478144 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚92191743354368 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚6285007 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚69017262100992 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105698816 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659456 VEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 975918963712 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚8968075 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 % _timestamp‚1.7844611337942135e+09 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10144 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚21403110 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786422 trainer/global_step‚1199 _runtime‚ 808.183735248 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884171 YHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 973217423473 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚93167768016896 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚92191760393728 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10144 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚11990595 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚92191760393728 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 _step‚95 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10142 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚69017356210688 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301876736 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105698816 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚2683064 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1005656627200 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚93167662318080 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659456 YHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 975918963712 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372523008 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚11990595_†QE0É+ _wandbruntime‚808  _runtime‚808$6ÿjÊ&ÊÄM UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚9410000 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10158 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚92266910060032 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚188 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚70079964199936 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚69073399828992 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚93243597700096 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚92266893020672 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12373 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2684978 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825894 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚9410002 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚6290063 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚ 808.598142208 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚21420573 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚92162380433055 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 974000918073 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300224 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105705984 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚12000415 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301876736 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1006332435456 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚93243703406080 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12302 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚184 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300224 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚8975229 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372523008 trainer/global_step‚1199 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 976793346048 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10156 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1006470261248 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚9410002 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚12010571 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚6290067 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚91188292728448 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚93243703406080 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚8975041 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10156 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 974087704607 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 ODeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884171 % _timestamp‚1.7844611342096472e+09 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105705984¢ØƒmÎ&ËÈM ODeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884171 % _timestamp‚1.7844611342096472e+09 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105705984 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚9410000 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10158 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚92266910060032 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚188 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚70079964199936 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚69073399828992 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚93243597700096 YFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚92266893020672 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12373 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2684978 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825894 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚9410002 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚6290063 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚ 808.598142208 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚21420573 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 VCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚92162380433055 WFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 974000918073 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300224 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105705984 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚12000415 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301876736 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1006332435456 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚93243703406080 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12302 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚184 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300224 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚8975229 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372523008 trainer/global_step‚1199 [JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 976793346048 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10156 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1006470261248 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚9410002 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚12010571 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚6290067 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚91188292728448 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚93243703406080 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚8975041 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10156 [JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 974087704607 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130`6 ½R0Ì+ _wandbruntime‚808  _runtime‚808¶c›Íý 0loss/FRt1/assignment_loss‚3.6812273925876027 0loss/FRt2/assignment_loss‚1.9325366850743846 2loss/SRqqt1/assignment_loss‚1.4325531486718528 1loss/FBt1/assignment_loss‚0.16985920523501416 0loss/FBt2/assignment_loss‚0.3035187104938015 1loss/SRqqt1/detection_loss‚2.3327680313746137 /loss/FBt1/detection_loss‚2.0477005637299337 .loss/FBt2/detection_loss‚4.207679623783362 /loss/FRt1/detection_loss‚0.5466076463885369 %loss/total_loss‚19.95555302210614 epoch‚2 trainer/global_step‚1199 _runtime‚ 808.598776459 _step‚97 2loss/SRqqt2/assignment_loss‚1.1067065227045012 /loss/FRt2/detection_loss‚1.4101949569887662 1loss/SRqqt2/detection_loss‚0.7842005350737712 % _timestamp‚1.7844611342107282e+09à³…‰Î 0loss/FRt1/assignment_loss‚3.6812273925876027 0loss/FRt2/assignment_loss‚1.9325366850743846 2loss/SRqqt1/assignment_loss‚1.4325531486718528 1loss/FBt1/assignment_loss‚0.16985920523501416 0loss/FBt2/assignment_loss‚0.3035187104938015 1loss/SRqqt1/detection_loss‚2.3327680313746137 /loss/FBt1/detection_loss‚2.0477005637299337 .loss/FBt2/detection_loss‚4.207679623783362 /loss/FRt1/detection_loss‚0.5466076463885369 %loss/total_loss‚19.95555302210614 epoch‚2 trainer/global_step‚1199 _runtime‚ 808.598776459 _step‚97 2loss/SRqqt2/assignment_loss‚1.1067065227045012 /loss/FRt2/detection_loss‚1.4101949569887662 1loss/SRqqt2/detection_loss‚0.7842005350737712 % _timestamp‚1.7844611342107282e+09a[HçhÏ‚(:Ý ÏîòÒžšÓ¤ disk.dm-1.in‚0 disk.rbd0.in‚0" disk.nvme0c0n1.in‚ 1852.4453125# proc.memory.availableMB‚20.3125 network.sent‚ 1.613872e+06 disk.nvme0n1.in‚ 1852.4453125 disk.md0.in‚0 disk.md0.out‚ 21.421875$ memory_percent‚99.87602233886719* proc.memory.percent‚17.571592330932617 cpu‚25.74795670313336 network.recv‚259234% disk./.usageGB‚1173.0916442871094% disk.nvme0n1.out‚15340.7392578125 disk.nvme1c1n1.in‚ 129.765625$ disk.nvme1n1.out‚10943.908203125 disk.dm-0.in‚ 1982.2109375 disk.rbd0.out‚0' disk.nvme0c0n1.out‚15340.7392578125& disk.nvme1c1n1.out‚10943.908203125 disk.nvme1n1.in‚ 129.765625" disk.dm-0.out‚25451.7314453125 disk.dm-1.out‚ 786.978515625" proc.memory.rssMB‚ 2878.9296875 proc.cpu.threads‚17) disk./.usagePercent‚76.41046372168113q€{œÐ‚(:‘ ÏîòÒüò‹¥ gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚48.0 gpu.0.powerWatts‚114.524' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚38.17466666666667 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0]fü0Ñ+ _wandbruntime‚822  _runtime‚8221lp¬ Òì| @&Purity/2FRt*SRqqt1FBt/event_proportion‚0.0016043527284637094 +particle/specificity‚0.8872056603431702 9"Purity/*FRt0SRqqt*FBt/event_purity‚0.1434442698955536 :"Purity/*FRt1SRqqt0FBt/event_purity‚0.07847241312265396 :"Purity/0FRt*SRqqt*FBt/SRqqt_purity‚0.16429191827774048 7 Purity/2FRt1SRqqt*FBt/FBt_purity‚0.9736841917037964 jet/accuracy_1_of_5‚1 7 Purity/*FRt*SRqqt2FBt/FBt_purity‚0.8779283165931702 ,&Purity/1FRt2SRqqt2FBt/event_proportion‚0 ,jet/accuracy_4_of_4‚0.002801120514050126 9"Purity/*FRt*SRqqt1FBt/SRqqt_purity‚0.5257519483566284 =&Purity/*FRt1SRqqt*FBt/event_proportion‚0.3866024911403656 :"Purity/*FRt1SRqqt*FBt/SRqqt_purity‚0.14702120423316956 7 Purity/1FRt1SRqqt1FBt/FRt_purity‚0.4300529956817627 +jet/accuracy_2_of_3‚0.23848356306552887 particle/accuracy_2_of_2‚1 7 Purity/2FRt0SRqqt*FBt/FBt_purity‚0.9722222089767456 ?&Purity/*FRt*SRqqt2FBt/event_proportion‚0.010997953824698925 :"Purity/1FRt*SRqqt1FBt/event_purity‚0.16500848531723022 ("Purity/2FRt1SRqqt2FBt/event_purity‚0 jet/accuracy_4_of_5‚0 :"Purity/2FRt1SRqqt*FBt/SRqqt_purity‚0.15995700657367706 9"Purity/0FRt1SRqqt0FBt/event_purity‚0.1330351084470749 =&Purity/1FRt*SRqqt0FBt/event_proportion‚0.6105840802192688 >&Purity/1FRt0SRqqt1FBt/event_proportion‚0.00969587080180645 @&Purity/1FRt0SRqqt2FBt/event_proportion‚0.0024879092816263437 9"Purity/2FRt*SRqqt1FBt/SRqqt_purity‚0.7824074029922485 <&Purity/*FRt*SRqqt0FBt/event_proportion‚0.881045401096344 9"Purity/*FRt*SRqqt2FBt/event_purity‚0.6416487693786621 ("Purity/0FRt2SRqqt1FBt/event_purity‚0 @&Purity/1FRt*SRqqt2FBt/event_proportion‚0.0027204242069274187 =&Purity/1FRt*SRqqt*FBt/event_proportion‚0.6297432780265808 :"Purity/1FRt1SRqqt0FBt/SRqqt_purity‚0.11464042961597443 (particle/accuracy‚0.8044899702072144 'particle/f_score‚0.5244719386100769 9"Purity/*FRt1SRqqt1FBt/SRqqt_purity‚0.5571820735931396 =&Purity/2FRt0SRqqt*FBt/event_proportion‚0.0208565853536129 ("Purity/2FRt1SRqqt0FBt/event_purity‚0 /particle/accuracy_3_of_3‚0.9985920190811157 ("Purity/2FRt1SRqqt2FBt/SRqqt_purity‚0 particle/accuracy_3_of_5‚0 8 Purity/1FRt0SRqqt0FBt/FRt_purity‚0.07467625290155411 particle/accuracy_2_of_3‚1 * Purity/2FRt*SRqqt2FBt/FBt_purity‚0.875 9"Purity/2FRt*SRqqt0FBt/SRqqt_purity‚0.1765838861465454 9"Purity/*FRt*SRqqt2FBt/SRqqt_purity‚0.5462963581085205 =&Purity/0FRt0SRqqt*FBt/event_proportion‚0.0390159972012043 +jet/accuracy_2_of_4‚0.23777253925800323 9"Purity/2FRt0SRqqt2FBt/event_purity‚0.0714285746216774 particle/accuracy_1_of_5‚1 =&Purity/*FRt0SRqqt1FBt/event_proportion‚0.0418526791036129 9"Purity/0FRt1SRqqt1FBt/eve/·°‚Æ3nt_purity‚0.3208686113357544 ;"Purity/0FRt2SRqqt0FBt/event_purity‚0.018289437517523766 :"Purity/1FRt0SRqqt*FBt/event_purity‚0.08046628534793854 7 Purity/0FRt1SRqqt2FBt/FBt_purity‚0.5192307829856873 ;"Purity/1FRt1SRqqt*FBt/event_purity‚0.008709171786904335 7 Purity/2FRt*SRqqt1FBt/FRt_purity‚0.2824999988079071 ("Purity/0FRt1SRqqt2FBt/event_purity‚0 8 Purity/*FRt*SRqqt0FBt/FRt_purity‚0.17859435081481934 :"Purity/*FRt2SRqqt0FBt/SRqqt_purity‚0.23088809847831726 7 Purity/*FRt2SRqqt1FBt/FRt_purity‚0.3499999940395355 :"Purity/0FRt*SRqqt*FBt/event_purity‚0.24215319752693176 7 Purity/0FRt*SRqqt2FBt/FBt_purity‚0.8684307932853699 9"Purity/0FRt*SRqqt1FBt/SRqqt_purity‚0.4858344495296478 >&Purity/0FRt2SRqqt*FBt/event_proportion‚0.02457682229578495 =&Purity/2FRt*SRqqt0FBt/event_proportion‚0.0382486991584301 <&Purity/*FRt2SRqqt*FBt/event_proportion‚0.054222472012043 & Purity/2FRt0SRqqt1FBt/FBt_purity‚1 9"Purity/0FRt0SRqqt*FBt/event_purity‚0.9474654197692871 ("Purity/1FRt2SRqqt0FBt/event_purity‚0 8 Purity/2FRt0SRqqt*FBt/FRt_purity‚0.06961995363235474 9"Purity/0FRt1SRqqt2FBt/SRqqt_purity‚0.6538461446762085 :"Purity/1FRt*SRqqt0FBt/SRqqt_purity‚0.14757616817951202 =&Purity/2FRt1SRqqt*FBt/event_proportion‚0.0127883180975914 7 Purity/2FRt0SRqqt2FBt/FRt_purity‚0.3571428656578064 =&Purity/*FRt0SRqqt*FBt/event_proportion‚0.5129278302192688 7 Purity/1FRt*SRqqt0FBt/FRt_purity‚0.1816055029630661 7 Purity/1FRt1SRqqt2FBt/FBt_purity‚0.6666666865348816 ?&Purity/2FRt1SRqqt0FBt/event_proportion‚0.011928013525903225 9"Purity/2FRt1SRqqt1FBt/SRqqt_purity‚0.7824074029922485 /particle/accuracy_4_of_4‚0.8873162269592285 7 Purity/*FRt*SRqqt1FBt/FRt_purity‚0.4041590094566345 7 Purity/*FRt0SRqqt*FBt/FRt_purity‚0.0840381532907486 8 Purity/*FRt2SRqqt0FBt/FRt_purity‚0.29571986198425293 9"Purity/0FRt0SRqqt2FBt/event_purity‚0.7760453820228577 & Purity/2FRt1SRqqt2FBt/FRt_purity‚0 7 Purity/*FRt0SRqqt1FBt/FBt_purity‚0.9885520935058594 @&Purity/*FRt2SRqqt1FBt/event_proportion‚0.0009998140158131719 :"Purity/0FRt*SRqqt0FBt/SRqqt_purity‚0.14738495647907257 ;"Purity/2FRt0SRqqt0FBt/event_purity‚0.005483916960656643 particle/accuracy_1_of_4‚1 particle/accuracy_2_of_4‚1 jet/accuracy_3_of_5‚0 epoch‚2 & Purity/2FRt1SRqqt1FBt/FBt_purity‚1 ,jet/accuracy_3_of_4‚0.022955847904086113 6validation_average_jet_accuracy‚0.1670665591955185 6 Purity/*FRt0SRqqt2FBt/FRt_purity‚0.629176139831543 =&Purity/1FRt0SRqqt*FBt/event_proportion‚0.4530552327632904 7 Purity/1FRt0SRqqt2FBt/FBt_purity‚0.9259259104728699 =&Purity/1FRt1SRqqt*FBt/event_proportion‚0.1534598171710968 8 Purity/2FRt*SRqqt0FBt/FRt_purity‚0.15473909676074982 7 Purity/2FRt1SRqqt*FBt/FRt_purity‚0.3062460422515869 :"Purity/*FRt1SRqqt*FBt/event_purity‚0.08477552980184555 7 Purity/*FRt1SRqqt2FBt/FRt_purity‚0.6499999761581421 ("Purity/0FRt0SRqqt0FBt/event_purity‚1 9"Purity/0FRt0SRqqt1FBt/event_purity‚0.9870335459709167 7 Purity/1FRt2SRqqt*FBt/FRt_purity‚0.3672681152820587 7 Purity/0FRt0SRqqt1FBt/FBt_purity‚0.9870335459709167 6 Purity/1FRt*SRqqt1FBt/FRt_purity‚0.426180899143219 >&Purity/1FRt2SRqqt*FBt/event_proportion‚0.02322823740541935 & Purity/1FRt2SRqqt1FBt/FBt_purity‚1 ;"Purity/2FRt0SRqqt*FBt/event_purity‚0.008630704134702682 particle/accuracy_4_of_5‚0 7 Purity/*FRt1SRqqt1FBt/FBt_purity‚0.9874683618545532 =&Purity/0FRt0SRqqt0FBt/event_proportion‚0.0462472103536129 ("Purity/2FRt1SRqqt*FBt/event_purity‚0 ("Purity/2FRt2SRqqt0FBt/event_purity‚0 ?&Purity/2FRt2SRqqt0FBt/event_proportion‚0.006417410913854837 particle/accuracy_1_of_3‚1 7 Purity/*FRt1SRqqt*FBt/FRt_purity‚0.4491008520126343 _runtime‚ 822.083984907 =&Purity/*FRt0SRqqt0FBt/event_proportion‚0.4607747495174408 =&Purity/*FRt1SRqqt0FBt/event_proportion‚0.3670479953289032 & Purity/*FRt2SRqqt1FBt/FBt_purity‚1 8&Purity/1FRt2SRqqt0FBt/event_proportion‚ 0.02294921875 ("Purity/1FRt2SRqqt1FBt/event_purity‚0 8 Purity/*FRt0SRqqt0FBt/FRt_purity‚0.07319771498441696 8 Purity/*FRt2SRqqt*FBt/FRt_purity‚0.29593944549560547 7 Purity/0FRt1SRqqt*FBt/FBt_purity‚0.9525397419929504 :"Purity/1FRt2SRqqt0FBt/SRqqt_purity‚0.25258246064186096 + Purity/2FRt*SRqqt2FBt/FRt_purity‚0.3125 ("Purity/2FRt*SRqqt2FBt/SRqqt_purity‚0 >&Purity/0FRt2SRqqt0FBt/event_proportion‚0.02385602705180645 ,&Purity/0FRt2SRqqt2FBt/event_proportion‚0 9"Purity/1FRt*SRqqt0FBt/event_purity‚0.0556122250854969 ;"Purity/1FRt1SRqqt1FBt/event_purity‚0.040314942598342896 8 Purity/2FRt*SRqqt*FBt/FRt_purity‚0.16159312427043915 7 Purity/2FRt1SRqqt0FBt/FRt_purity‚0.3104703426361084 @&Purity/2FRt1SRqqt1FBt/event_proportion‚0.0008370535797439516 +jet/accuracy_1_of_1‚0.11730758100748062 :"Purity/*FRt2SRqqt1FBt/SRqqt_purity‚0.25231483578681946 7 Purity/0FRt*SRqqt*FBt/FBt_purity‚0.9553054571151733 & Purity/2FRt*SRqqt1FBt/FBt_purity‚1 @&Purity/2FRt1SRqqt2FBt/event_proportion‚2.325148852833081e-05 jet/accuracy_5_of_5‚0 :"Purity/*FRt2SRqqt*FBt/SRqqt_purity‚0.23115669190883636 9"Purity/0FRt2SRqqt0FBt/SRqqt_purity‚0.2119097262620926 :"Purity/0FRt1SRqqt*FBt/event_purity‚0.14251133799552917 ?&Purity/0FRt1SRqqt1FBt/event_proportion‚0.011555989272892475 >&Purity/1FRt*SRqqt1FBt/event_proportion‚0.01643880270421505 8 Purity/1FRt2SRqqt0FBt/FRt_purity‚0.36802616715431213 *jet/accuracy_1_of_2‚0.5128798484802246 -"Purity/2FRt*SRqqt2FBt/event_purity‚0.0625 8 Purity/*FRt1SRqqt0FBt/FRt_purity‚0.45123305916786194 9"Purity/*FRt2SRqqt0FBt/event_purity‚0.0081094428896904 9"Purity/1FRt*SRqqt*FBt/SRqqt_purity‚0.1631682813167572 7 Purity/1FRt1SRqqt*FBt/FBt_purity‚0.9773953557014465 *particle/sensitivity‚0.502061128616333 7 Purity/2FRt0SRqqt1FBt/FRt_purity‚0.3229166865348816 particle/accuracy_5_of_5‚0 ?&Purity/0FRt*SRqqt2FBt/event_proportion‚0.008068266324698925 @&Purity/1FRt1SRqqt1FBt/event_proportion‚0.0064639137126505375 ?&Purity/2FRt2SRqqt*FBt/event_proportion‚0.006417410913854837 *jet/accuracy_1_of_3‚0.7115623354911804 jet/accuracy_2_of_5‚0 7 Purity/*FRt*SRqqt2FBt/FRt_purity‚0.6296296119689941 8&Purity/*FRt2SRqqt0FBt/event_proportion‚ 0.05322265625 0"Purity/0FRt2SRqqt1FBt/SRqqt_purity‚ 0.2109375 9"Purity/1FRt*SRqqt2FBt/SRqqt_purity‚0.3888888955116272 7 Purity/2FRt1SRqqt1FBt/FRt_purity‚0.2708333432674408 :"Purity/2FRt2SRqqt*FBt/SRqqt_purity‚0.23329557478427887 :"Purity/2FRt*SRqqt1FBt/event_purity‚0.02500000037252903 7 Purity/*FRt*SRqqt1FBt/FBt_purity‚0.9886044263839722 =&Purity/0FRt1SRqqt*FBt/event_proportion‚0.2203543484210968 9"Purity/1FRt*SRqqt1FBt/SRqqt_purity‚0.5867950320243835 7 Purity/1FRt0SRqqt*FBt/FBt_purity‚0.9712649583816528 :"Purity/2FRt2SRqqt0FBt/SRqqt_purity‚0.23329557478427887 +jet/accuracy_3_of_3‚0.02388635091483593 7 Purity/*FRt*SRqqt*FBt/FRt_purity‚0.1868348866701126 9"Purity/*FRt*SRqqt1FBt/event_purity‚0.6049481630325317 8 Purity/*FRt0SRqqt1FBt/FRt_purity‚0.40682607889175415 ("Purity/*FRt1SRqqt2FBt/event_purity‚0 =&Purity/0FRt*SRqqt*FBt/event_proportion‚0.2839471697807312 A&Purity/2FRt0SRqqt2FBt/event_proportion‚0.00018601190822664648 7 Purity/*FRt1SRqqt1FBt/FRt_purity‚0.4016929268836975 :"Purity/1FRt*SRqqt2FBt/event_purity‚0.41991686820983887 9 Purity/2FRt0SRqqt0FBt/FRt_purity‚0.056759245693683624 =&Purity/*FRt1SRqqt1FBt/event_proportion‚0.0188569575548172 :"Purity/0FRt*SRqqt0FBt/event_purity‚0.12121870368719101 & Purity/0FRt2SRqqt1FBt/FBt_purity‚1 7 Purity/1FRt*SRqqt2FBt/FRt_purity‚0.6638510823249817 ("Purity/1FRt2SRqqt*FBt/event_purity‚0 <"Purity/2FRt*SRqqt*FBt/event_purity‚0.0046581486240029335 9"Purity/*FRt0SRqqt1FBt/event_purity‚0.7990488409996033 :"Purity/*FRt*SRqqt*FBt/SRqqt_purity‚0.16533531248569489 9"Purity/*FRt1SRqqt1FBt/event_purity‚0.2088833749294281 7 Purity/1FRt*SRqqt*FBt/FRt_purity‚0.1901000738143921 :"Purity/0FRt2SRqqt*FBt/SRqqt_purity‚0.21209698915481567 *loss/val_total_loss‚19.212646764698263 =&Purity/*FRt*SRqqt*FBt/event_proportion‚0.9537528157234192 9"Purity/*FRt1SRqqt2FBt/SRqqt_purity‚0.5462963581085205 6 Purity/1FRt*SRqqt1FBt/FBt_purity‚0.994339644908905 7 Purity/1FRt2SRqqt1FBt/FRt_purity‚0.3499999940395355 ;"Purity/*FRt2SRqqt*FBt/event_purity‚0.007969309575855732 9"Purity/0FRt*SRqqt1FBt/event_purity‚0.7915863990783691 7 Purity/0FRt0SRqqt2FBt/FBt_purity‚0.8880225419998169 9"Purity/1FRt0SRqqt2FBt/event_purity‚0.4514928162097931 >&Purity/2FRt0SRqqt0FBt/event_proportion‚0.01990327425301075 *jet/accuracy_1_of_4‚0.6960964798927307 =&Purity/*FRt0SRqqt2FBt/event_proportion‚0.0103004090487957 :"Purity/0FRt1SRqqt*FBt/SRqqt_purity‚0.15404582023620605 & Purity/0FRt2SRqqt*FBt/FBt_purity‚1 @&Purity/0FRt2SRqqt1FBt/event_proportion‚0.0007207961170934141 ("Purity/1FRt1SRqqt2FBt/event_purity‚0 @&Purity/2FRt0SRqqt1FBt/event_proportion‚0.0007672990905120969 :"Purity/*FRt1SRqqt0FBt/SRqqt_purity‚0.12527161836624146 =&Purity/0FRt1SRqqt0FBt/event_proportion‚0.2083565890789032 ;"Purity/1FRt*SRqqt*FBt/event_purity‚0.059962522238492966 A&Purity/1FRt2SRqqt1FBt/event_proportion‚0.00027901786961592734 9"Purity/0FRt*SRqqt2FBt/SRqqt_purity‚0.6538461446762085 9"Purity/*FRt0SRqqt2FBt/event_purity‚0.6854254007339478 7 Purity/*FRt1SRqqt2FBt/FBt_purity‚0.5740740299224854 9"Purity/0FRt1SRqqt1FBt/SRqqt_purity‚0.5197550058364868 7 Purity/1FRt*SRqqt*FBt/FBt_purity‚0.9724316596984863 7 Purity/*FRt0SRqqt2FBt/FBt_purity‚0.8985686302185059 =&Purity/0FRt0SRqqt1FBt/event_proportion‚0.0313895083963871 :"Purity/1FRt0SRqqt0FBt/event_purity‚0.07467625290155411 7 Purity/1FRt0SRqqt2FBt/FRt_purity‚0.6532502174377441 ("Purity/2FRt1SRqqt1FBt/event_purity‚0 =&Purity/0FRt*SRqqt0FBt/event_proportion‚0.2322126179933548 9"Purity/1FRt1SRqqt2FBt/SRqqt_purity‚0.3888888955116272 7 Purity/*FRt*SRqqt*FBt/FBt_purity‚0.9602847099304199 9"Purity/*FRt*SRqqt0FBt/event_purity‚0.0706205889582634 9"Purity/0FRt*SRqqt2FBt/event_purity‚0.7325327396392822 @&Purity/0FRt1SRqqt2FBt/event_proportion‚0.0004417782765813172 7 Purity/1FRt0SRqqt1FBt/FBt_purity‚0.9918831586837769 @&Purity/1FRt1SRqqt2FBt/event_proportion‚0.0002325148816453293 8 Purity/1FRt0SRqqt*FBt/FRt_purity‚0.08533980697393417 ( Purity/2FRt1SRqqt2FBt/FBt_purity‚0.5 ;"Purity/1FRt1SRqqt0FBt/event_purity‚0.007203222718089819 7 Purity/1FRt1SRqqt2FBt/FRt_purity‚0.7222222089767456 @&Purity/2FRt*SRqqt2FBt/event_proportion‚0.0002092633949359879 ,jet/accuracy_2_of_2‚0.059144485741853714 /particle/accuracy_3_of_4‚0.9979296326637268 _step‚98 9"Purity/1FRt2SRqqt*FBt/SRqqt_purity‚0.2534150183200836 :"Purity/2FRt*SRqqt*FBt/SRqqt_purity‚0.19515500962734222 9"Purity/2FRt1SRqqt0FBt/SRqqt_purity‚0.1174362525343895 particle/accuracy_1_of_2‚1 7 Purity/2FRt0SRqqt2FBt/FBt_purity‚0.9285714030265808 :"Purity/*FRt*SRqqt*FBt/event_purity‚0.11192169785499573 7 Purity/1FRt1SRqqt1FBt/FBt_purity‚0.9970238208770752 8 Purity/2FRt2SRqqt*FBt/FRt_purity‚0.16317085921764374 $ _timestamp‚1.784461147695108e+09 :"Purity/0FRt2SRqqt*FBt/event_purity‚0.01777801848948002 8 Purity/1FRt1SRqqt*FBt/FRt_purity‚0.47315648198127747 =&Purity/1FRt1SRqqt0FBt/event_proportion‚0.1467633992433548 ;"Purity/2FRt*SRqqt0FBt/event_purity‚0.003039250150322914 :"Purity/*FRt0SRqqt0FBt/event_purity‚0.07170537859201431 @&Purity/*FRt1SRqqt2FBt/event_proportion‚0.0006975446594879031 & Purity/*FRt2SRqqt*FBt/FBt_purity‚1 =&Purity/1FRt0SRqqt0FBt/event_proportion‚0.4408714771270752 ("Purity/2FRt2SRqqt*FBt/event_purity‚0 ,&Purity/2FRt2SRqqt1FBt/event_proportion‚0 =&Purity/0FRt*SRqqt1FBt/event_proportion‚0.0436662957072258 7 Purity/0FRt1SRqqt1FBt/FBt_purity‚0.9822424054145813 9"Purity/0FRt1SRqqt0FBt/SRqqt_purity‚0.1330351084470749 7 Purity/0FRt*SRqqt1FBt/FBt_purity‚0.9859228730201721 ?&Purity/0FRt0SRqqt2FBt/event_proportion‚0.007626487873494625 7 Purity/2FRt*SRqqt*FBt/FBt_purity‚0.9750000238418579 ("Purity/*FRt2SRqqt1FBt/event_purity‚0 ,&Purity/2FRt2SRqqt2FBt/event_proportion‚0 ,&Purity/*FRt2SRqqt2FBt/event_proportion‚0 7 Purity/0FRt0SRqqt*FBt/FBt_purity‚0.9560576677322388 9"Purity/1FRt1SRqqt1FBt/SRqqt_purity‚0.6078368425369263 & Purity/1FRt2SRqqt*FBt/FBt_purity‚1 particle/accuracy_1_of_1‚1 9"Purity/*FRt*SRqqt0FBt/SRqqt_purity‚0.1489531695842743 7 Purity/*FRt1SRqqt*FBt/FBt_purity‚0.9593394994735718 7 Purity/1FRt*SRqqt2FBt/FBt_purity‚0.9026550054550171 7 Purity/1FRt1SRqqt0FBt/FRt_purity‚0.4744177758693695 <&Purity/2FRt*SRqqt*FBt/event_proportion‚0.040062315762043 9"Purity/2FRt0SRqqt1FBt/event_purity‚0.0520833358168602 =&Purity/*FRt*SRqqt1FBt/event_proportion‚0.0617094486951828 :"Purity/1FRt2SRqqt1FBt/SRqqt_purity‚0.30000001192092896 8 Purity/2FRt2SRqqt0FBt/FRt_purity‚0.16317085921764374 trainer/global_step‚1199 7 Purity/1FRt0SRqqt1FBt/FRt_purity‚0.4218703806400299 particle/accuracy_2_of_5‚1 7 Purity/*FRt0SRqqt*FBt/FBt_purity‚0.9598701000213623 :"Purity/1FRt0SRqqt1FBt/event_purity‚0.24779640138149261 :"Purity/1FRt1SRqqt*FBt/SRqqt_purity‚0.13579580187797546ÞÀôdv>Óð| 9"Purity/0FRt1SRqqt1FBt/SRqqt_purity‚0.5197550058364868 7 Purity/1FRt*SRqqt*FBt/FBt_purity‚0.9724316596984863 7 Purity/*FRt0SRqqt2FBt/FBt_purity‚0.8985686302185059 =&Purity/0FRt0SRqqt1FBt/event_proportion‚0.0313895083963871 :"Purity/1FRt0SRqqt0FBt/event_purity‚0.07467625290155411 7 Purity/1FRt0SRqqt2FBt/FRt_purity‚0.6532502174377441 ("Purity/2FRt1SRqqt1FBt/event_purity‚0 =&Purity/0FRt*SRqqt0FBt/event_proportion‚0.2322126179933548 9"Purity/1FRt1SRqqt2FBt/SRqqt_purity‚0.3888888955116272 7 Purity/*FRt*SRqqt*FBt/FBt_purity‚0.9602847099304199 9"Purity/*FRt*SRqqt0FBt/event_purity‚0.0706205889582634 9"Purity/0FRt*SRqqt2FBt/event_purity‚0.7325327396392822 @&Purity/0FRt1SRqqt2FBt/event_proportion‚0.0004417782765813172 7 Purity/1FRt0SRqqt1FBt/FBt_purity‚0.9918831586837769 @&Purity/1FRt1SRqqt2FBt/event_proportion‚0.0002325148816453293 8 Purity/1FRt0SRqqt*FBt/FRt_purity‚0.08533980697393417 ( Purity/2FRt1SRqqt2FBt/FBt_purity‚0.5 ;"Purity/1FRt1SRqqt0FBt/event_purity‚0.007203222718089819 7 Purity/1FRt1SRqqt2FBt/FRt_purity‚0.7222222089767456 @&Purity/2FRt*SRqqt2FBt/event_proportion‚0.0002092633949359879 ,jet/accuracy_2_of_2‚0.059144485741853714 /particle/accuracy_3_of_4‚0.9979296326637268 _step‚98 9"Purity/1FRt2SRqqt*FBt/SRqqt_purity‚0.2534150183200836 :"Purity/2FRt*SRqqt*FBt/SRqqt_purity‚0.19515500962734222 9"Purity/2FRt1SRqqt0FBt/SRqqt_purity‚0.1174362525343895 particle/accuracy_1_of_2‚1 7 Purity/2FRt0SRqqt2FBt/FBt_purity‚0.9285714030265808 :"Purity/*FRt*SRqqt*FBt/event_purity‚0.11192169785499573 7 Purity/1FRt1SRqqt1FBt/FBt_purity‚0.9970238208770752 8 Purity/2FRt2SRqqt*FBt/FRt_purity‚0.16317085921764374 $ _timestamp‚1.784461147695108e+09 :"Purity/0FRt2SRqqt*FBt/event_purity‚0.01777801848948002 8 Purity/1FRt1SRqqt*FBt/FRt_purity‚0.47315648198127747 =&Purity/1FRt1SRqqt0FBt/event_proportion‚0.1467633992433548 ;"Purity/2FRt*SRqqt0FBt/event_purity‚0.003039250150322914 :"Purity/*FRt0SRqqt0FBt/event_purity‚0.07170537859201431 @&Purity/*FRt1SRqqt2FBt/event_proportion‚0.0006975446594879031 & Purity/*FRt2SRqqt*FBt/FBt_purity‚1 =&Purity/1FRt0SRqqt0FBt/event_proportion‚0.4408714771270752 ("Purity/2FRt2SRqqt*FBt/event_purity‚0 ,&Purity/2FRt2SRqqt1FBt/event_proportion‚0 =&Purity/0FRt*SRqqt1FBt/event_proportion‚0.0436662957072258 7 Purity/0FRt1SRqqt1FBt/FBt_purity‚0.9822424054145813 9"Purity/0FRt1SRqqt0FBt/SRqqt_purity‚0.1330351084470749 7 Purity/0FRt*SRqqt1FBt/FBt_purity‚0.9859228730201721 ?&Purity/0FRt0SRqqt2FBt/event_proportion‚0.007626487873494625 7 Purity/2FRt*SRqqt*FBt/FBt_purity‚0.9750000238418579 ("Purity/*FRt2SRqqt1FBt/event_purity‚0 ,&Purity/2FRt2SRqqt2FBt/event_proportion‚0 ,&Purity/*FRt2SRqqt2FBt/event_proportion‚0 7 Purity/0FRt0SRqqt*FBt/FBt_purity‚0.9560576677322388 9"Purity/1FRt1SRqqt1FBt/SRqqt_purity‚0.6078368425369263 & Purity/1FRt2SRqqt*FBt/FBt_purity‚1 particle/accuracy_1_of_1‚1 9"Purity/*FRt*SRqqt0FBt/SRqqt_purity‚0.1489531695842743 7 Purity/*FRt1SRqqt*FBt/FBt_purity‚0.9593394994735718 7 Purity/1FRt*SRqqt2FBt/FBt_purity‚0.9026550054550171 7 Purity/1FRt1SRqqt0FBt/FRt_purity‚0.4744177758693695 <&Purity/2FRt*SRqqt*FBt/event_proportion‚0.040062315762043 9"Purity/2FRt0SRqqt1FBt/event_purity‚0.0520833358168602 =&Purity/*FRt*SRqqt1FBt/event_proportion‚0.0617094486951828 :"Purity/1FRt2SRqqt1FBt/SRqqt_purity‚0.30000001192092896 8 Purity/2FRt2SRqqt0FBt/FRt_purity‚0.16317085921764374 trainer/global_step‚1199 7 Purity/1FRt0SRqqt1FBt/FRt_purity‚0.4218703806400299 particle/accuracy_2_of_5‚1 7 Purity/*FRt0SRqqt*FBt/FBt_purity‚0.9598701000213623 :"Purity/1FRt0SRqqt1FBt/event_purity‚0.24779640138149261 :"Purity/1FRt1SRqqt*FBt/SRqqt_purity‚0.13579580187797546 @&Purity/2FRt*SRqqt1FBt/event_proportion‚0.0016043527284637094 +particle/specificity‚0.8872056603431702 9"Purity/*FRt0SRqqt*FBt/event_purity‚0.1434442698955536 :"Purity/*FRt1SRqqt0FBt/event_purity‚0.07847241312265396 :"Purity/0FRt*SRqqt*FBt/SRqqt_purity‚0.16429191827774048 7 Purity/2FRt1SRqqt*FBt/FBt_purity‚0.9736841917037964 jet/accuracy_1_of_5‚1 7 Purity/*FRt*SRqqt2FBt/FBt_purity‚0.8779283165931702 ,&Purity/1FRt2SRqqt2FBt/event_proportion‚0 ,jet/accuracy_4_of_4‚0.002801120514050126 9"Purity/*FRt*SRqqt1FBt/SRqqt_purity‚0.5257519483566284 =&Purity/*FRt1SRqqt*FBt/event_proportion‚0.3866024911403656 :"Purity/*FRt1SRqqt*FBt/SRqqt_purity‚0.14702120423316956 7 Purity/1FRt1SRqqt1FBt/FRt_purity‚0.4300529956817627 +jet/accuracy_2_of_3‚0.23848356306552887 particle/accuracy_2_of_2‚1 7 Purity/2FRt0SRqqt*FBt/FBt_purity‚0.9722222089767456 ?&Purity/*FRt*SRqqt2FBt/event_proportion‚0.010997953824698925 :"Purity/1FRt*SRqqt1FBt/event_purity‚0.16500848531723022 ("Purity/2FRt1SRqqt2FBt/event_purity‚0 jet/accuracy_4_of_5‚0 :"Purity/2FRt1SRqqt*FBt/SRqqt_purity‚0.15995700657367706 9"Purity/0FRt1SRqqt0FBt/event_purity‚0.1330351084470749 =&Purity/1FRt*SRqqt0FBt/event_proportion‚0.6105840802192688 >&Purity/1FRt0SRqqt1FBt/event_proportion‚0.00969587080180645 @&Purity/1FRt0SRqqt2FBt/event_proportion‚0.0024879092816263437 9"Purity/2FRt*SRqqt1FBt/SRqqt_purity‚0.7824074029922485 <&Purity/*FRt*SRqqt0FBt/event_proportion‚0.881045401096344 9"Purity/*FRt*SRqqt2FBt/event_purity‚0.6416487693786621 ("Purity/0FRt2SRqqt1FBt/event_purity‚0 @&Purity/1FRt*SRqqt2FBt/event_proportion‚0.0027204242069274187 =&Purity/1FRt*SRqqt*FBt/event_proportion‚0.6297432780265808 :"Purity/1FRt1SRqqt0FBt/SRqqt_purity‚0.11464042961597443 (particle/accuracy‚0.8044899702072144 'particle/f_score‚0.5244719386100769 9"Purity/*FRt1SRqqt1FBt/SRqqt_purity‚0.5571820735931396 =&Purity/2FRt0SRqqt*FBt/event_proportion‚0.0208565853536129 ("Purity/2FRt1SRqqt0FBt/event_purity‚0 /particle/accuracy_3_of_3‚0.9985920190811157 ("Purity/2FRt1SRqqt2FBt/SRqqt_purity‚0 particle/accuracy_3_of_5‚0 8 Purity/1FRt0SRqqt0FBt/FRt_purity‚0.07467625290155411 particle/accuracy_2_of_3‚1 * Purity/2FRt*SRqqt2FBt/FBt_purity‚0.875 9"Purity/2FRt*SRqqt0FBt/SRqqt_purity‚0.1765838861465454 9"Purity/*FRt*SRqqt2FBt/SRqqt_purity‚0.5462963581085205 =&Purity/0FRt0SRqqt*FBt/event_proportion‚0.0390159972012043 +jet/accuracy_2_of_4‚0.23777253925800323 9"Purity/2FRt0SRqqt2FBt/event_purity‚0.0714285746216774 particle/accuracy_1_of_5‚1 =&Purity/*FRt0SRqqt1FBt/event_proportion‚0.0418526791036129 9"Purity/0FRt1SRqqt1FBt/event_purity‚0.3208686113357544 ;"Purity/0FRt2SRqqt0FBt/event_purity‚0.018289437517523766 :"Purity/1FRt0SRqqt*FBt/event_purity‚0.08046628534793854 7 Purity/0FRt1SRqqt2FBt/FBt_purity‚0.5192307829856873 ;"Purity/1FRt1SRqqt*FBt/event_purity‚0.008709171786904335 7 Purity/2FRt*SRqqt1FBt/FRt_purity‚0.2824999988079071 ("Purity/0FRt1SRqqt2FBt/event_purity‚0 8 Purity/*FRt*SRqqt0FBt/FRt_purity‚0.17859435081481934 :"Purity/*FRt2SRqqt0FBt/SRqqt_purity‚0.23088809847831726 7 Purity/*FRt2SRqqt1FBt/FRt_purity‚0.3499999940395355 :"Purity/0FRt*SRqqt*FBt/event_purity‚0.24215319752693176 7 Purity/0FRt*SRqqt2FBt/FBt_purity‚0.8684307932853699 9"Purity/0FRt*SRqqt1FBt/SRqqt_purity‚0.4858344495296478 >&Purity/0FRt2SRqqt*FBt/event_proportion‚0.02457682229578495 =&Purity/2FRt*SRqqt0FBt/event_proportion‚0.0382486991584301 <&Purity/*FRt2SRqqt*FBt/event_proportion‚0.054222472012043 & Purity/2FRt0SRqqt1FBt/FBt_purity‚1 9"Purity/0FRt0SRqqt*FBt/event_purity‚0.9474654197692871 ("Purity/1FRt2SRqqt0FBt/event_purity‚0 8 Purity/2FRt0SRqqt*FBt/FRt_purity‚0.06961995363235474 9"Purity/0FRt1SRqqt2FBt/SRqqt_purity‚0.6538461446762085 :"Purity/1FRt*SRqqt0FBt/SRqqt_purity‚0.14757616817951202 =&Purity/2FRt1SRqqt*FBt/event_proportion‚0.0127883180975914 7 Purity/2FRt0SRqqt2FBt/FRt_purity‚0.3571428656578064 =&Purity/*FRt0SRqqt*FBt/event_proportion‚0.5129278302192688 7 Purity/1FRt*SRqqt0FBt/FRt_purity‚0.1816055029630661 7 Purity/1FRt1SRqqt2FBt/FBt_purity‚0.6666666865348816 ?&Purity/2FRt1SRqqt0FBt/event_proportion‚0.011928013525903225 9"Purity/2FRt1SRqqt1FBt/SRqqt_purity‚0.7824074029922485 /particle/accuracy_4_of_4‚0.8873162269592285 7 Purity/*FRt*SRqqt1FBt/FRt_purity‚0.4041590094566345 7 Purity/*FRt0SRqqt*FBt/FRt_purity‚0.0840381532907486 8 Purity/*FRt2SRqqt0FBt/FRt_purity‚0.29571986198425293 9"Purity/0FRt0SRqqt2FBt/event_purity‚0.7760453820228577 & Purity/2FRt1SRqqt2FBt/FRt_purity‚0 7 Purity/*FRt0SRqqt1FBt/FBt_purity‚0.9885520935058594 @&Purity/*FRt2SRqqt1FBt/event_proportion‚0.0009998140158131719 :"Purity/0FRt*SRqqt0FBt/SRqqt_purity‚0.14738495647907257 ;"Purity/2FRt0SRqqt0FBt/event_purity‚0.005483916960656643 particle/accuracy_1_of_4‚1 particle/accuracy_2_of_4‚1 jet/accuracy_3_of_5‚0 epoch‚2 & Purity/2FRt1SRqqt1FBt/FBt_purity‚1 ,jet/accuracy_3_of_4‚0.022955847904086113 6validation_average_jet_accuracy‚0.1670665591955185 6 Purity/*FRt0SRqqt2FBt/FRt_purity‚0.629176139831543 =&Purity/1FRt0SRqqt*FBt/event_proportion‚0.4530552327632904 7 Purity/1FRt0SRqqt2FBt/FBt_purity‚0.9259259104728699 =&Purity/1FRt1SRqqt*FBt/event_proportion‚0.1534598171710968 8 Purity/2FRt*SRqqt0FBt/FRt_purity‚0.15473909676074982 7 Purity/2FRt1SRqqt*FBt/FRt_purity‚0.3062460422515869 :"Purity/*FRt1SRqqt*FBt/event_purity‚0.08477552980184555 7 Purity/*FRt1SRqqt2FBt/FRt_purity‚0.6499999761581421 ("Purity/0FRt0SRqqt0FBt/event_purity‚1 9"Purity/0FRt0SRqqt1FBt/event_purity‚0.9870335459709167 7 Purity/1FRt2SRqqt*FBt/FRt_purity‚0.3672681152820587 7 Purity/0FRt0SRqqt1FBt/FBt_purity‚0.9870335459709167 6 Purity/1FRt*SRqqt1FBt/FRt_purity‚0.426180899143219 >&Purity/1FRt2SRqqt*FBt/event_proportion‚0.02322823740541935 & Purity/1FRt2SRqqt1FBt/FBt_purity‚1 ;"Purity/2FRt0SRqqt*FBt/event_purity‚0.008630704134702682 particle/accuracy_4_of_5‚0 7 Purity/*FRt1SRqqt1FBt/FBt_purity‚0.9874683618545532 =&Purity/0FRt0SRqqt0FBt/event_proportion‚0.0462472103536129 ("Purity/2FRt1SRqqt*FBt/event_purity‚0 ("Purity/2FRt2SRqqt0FBt/event_purity‚0 ?&Purity/2FRt2SRqqt0FBt/event_proportion‚0.006417410913854837 particle/accuracy_1_of_3‚1 7 Purity/*FRt1SRqqt*FBt/FRt_purity‚0.4491008520126343 _runtime‚ 822.083984907 =&Purity/*FRt0SRqqt0FBt/event_proportion‚0.4607747495174408 =&Purity/*FRt1SRqqt0FBt/event_proportion‚0.3670479953289032 & Purity/*FRt2SRqqt1FBt/FBt_purity‚1 8&Purity/1FRt2SRqqt0FBt/event_proportion‚ 0.02294921875 ("Purity/1FRt2SRqqt1FBt/event_purity‚0 8 Purity/*FRt0SRqqt0FBt/FRt_purity‚0.07319771498441696 8 Purity/*FRt2SRqqt*FBt/FRt_purity‚0.29593944549560547 7 Purity/0FRt1SRqqt*FBt/FBt_purity‚0.9525397419929504 :"Purity/1FRt2SRqqt0FBt/SRqqt_purity‚0.25258246064186096 + Purity/2FRt*SRqqt2FBt/FRt_purity‚0.3125 ("Purity/2FRt*SRqqt2FBt/SRqqt_purity‚0 >&Purity/0FRt2SRqqt0FBt/event_proportion‚0.02385602705180645 ,&Purity/0FRt2SRqqt2FBt/event_proportion‚0 9"Purity/1FRt*SRqqt0FBt/event_purity‚0.0556122250854969 ;"Purity/1FRt1SRqqt1FBt/event_purity‚0.040314942598342896 8 Purity/2FRt*SRqqt*FBt/FRt_purity‚0.16159312427043915 7 Purity/2FRt1SRqqt0FBt/FRt_purity‚0.3104703426361084 @&Purity/2FRt1SRqqt1FBt/event_proportion‚0.0008370535797439516 +jet/accuracy_1_of_1‚0.11730758100748062 :"Purity/*FRt2SRqqt1FBt/SRqqt_purity‚0.25231483578681946 7 Purity/0FRt*SRqqt*FBt/FBt_purity‚0.9553054571151733 & Purity/2FRt*SRqqt1FBt/FBt_purity‚1 @&Purity/2FRt1SRqqt2FBt/event_proportion‚2.325148852833081e-05 jet/accuracy_5_of_5‚0 :"Purity/*FRt2SRqqt*FBt/SRqqt_purity‚0.23115669190883636 9"Purity/0FRt2SRqqt0FBt/SRqqt_purity‚0.2119097262620926 :"Purity/0FRt1SRqqt*FBt/event_purity‚0.14251133799552917 ?&Purity/0FRt1SRqqt1FBt/event_proportion‚0.011555989272892475 >&Purity/1FRt*SRqqt1FBt/event_proportion‚0.01643880270421505 8 Purity/1FRt2SRqqt0FBt/FRt_purity‚0.36802616715431213 *jet/accuracy_1_of_2‚0.5128798484802246 -"Purity/2FRt*SRqqt2FBt/event_purity‚0.0625 8 Purity/*FRt1SRqqt0FBt/FRt_purity‚0.45123305916786194 9"Purity/*FRt2SRqqt0FBt/event_purity‚0.0081094428896904 9"Purity/1FRt*SRqqt*FBt/SRqqt_purity‚0.1631682813167572 7 Purity/1FRt1SRqqt*FBt/FBt_purity‚0.9773953557014465 *particle/sensitivity‚0.502061128616333 7 Purity/2FRt0SRqqt1FBt/FRt_purity‚0.3229166865348816 particle/accuracy_5_of_5‚0 ?&Purity/0FRt*SRqqt2FBt/event_proportion‚0.008068266324698925 @&Purity/1FRt1SRqqt1FBt/event_proportion‚0.0064639137126505375 ?&Purity/2FRt2SRqqt*FBt/event_proportion‚0.006417410913854837 *jet/accuracy_1_of_3‚0.7115623354911804 jet/accuracy_2_of_5‚0 7 Purity/*FRt*SRqqt2FBt/FRt_purity‚0.6296296119689941 8&Purity/*FRt2SRqqt0FBt/event_proportion‚ 0.05322265625 0"Purity/0FRt2SRqqt1FBt/SRqqt_purity‚ 0.2109375 9"Purity/1FRt*SRqqt2FBt/SRqqt_purity‚0.3888888955116272 7 Purity/2FRt1SRqqt1FBt/FRt_purity‚0.2708333432674408 :"Purity/2FRt2SRqqt*FBt/SRqqt_purity‚0.23329557478427887 :"Purity/2FRt*SRqqt1FBt/event_purity‚0.02500000037252903 7 Purity/*FRt*SRqqt1FBt/FBt_purity‚0.9886044263839722 =&Purity/0FRt1SRqqt*FBt/event_proportion‚0.2203543484210968 9"Purity/1FRt*SRqqt1FBt/SRqqt_purity‚0.5867950320243835 7 Purity/1FRt0SRqqt*FBt/FBt_purity‚0.9712649583816528 :"Purity/2FRt2SRqqt0FBt/SRqqt_purity‚0.23329557478427887 +jet/accuracy_3_of_3‚0.02388635091483593 7 Purity/*FRt*SRqqt*FBt/FRt_purity‚0.1868348866701126 9"Purity/*FRt*SRqqt1FBt/event_purity‚0.6049481630325317 8 Purity/*FRt0SRqqt1FBt/FRt_purity‚0.40682607889175415 ("Purity/*FRt1SRqqt2FBt/event_purity‚0 =&Purity/0FRt*SRqqt*FBt/event_proportion‚0.2839471697807312 A&Purity/2FRt0SRqqt2FBt/event_proportion‚0.00018601190822664648 7 Purity/*FRt1SRqqt1FBt/FRt_purity‚0.4016929268836975 :"Purity/1FRt*SRqqt2FBt/event_purity‚0.41991686820983887 9 Purity/2FRt0SRqqt0FBt/FRt_purity‚0.056759245693683624 =&Purity/*FRt1SRqqt1FBt/event_proportion‚0.0188569575548172 :"Purity/0FRt*SRqqt0FBt/event_purity‚0.12121870368719101 & Purity/0FRt2SRqqt1FBt/FBt_purity‚1 7 Purity/1FRt*SRqqt2FBt/FRt_purity‚0.6638510823249817 ("Purity/1FRt2SRqqt*FBt/event_purity‚0 <"Purity/2FRt*SRqqt*FBt/event_purity‚0.0046581486240029335 9"Purity/*FRt0SRqqt1FBt/event_purity‚0.7990488409996033 :"Purity/*FRt*SRqqt*FBt/SRqqt_purity‚0.16533531248569489 9"Purity/*FRt1SRqqt1FBt/event_purity‚0.2088833749294281 7 Purity/1FRt*SRqqt*FBt/FRt_purity‚0.1901000738143921 :"Purity/0FRt2SRqqt*FBt/SRqqt_purity‚0.21209698915481567 *loss/val_total_loss‚19.212646764698263 =&Purity/*FRt*SRqqt*FBt/event_proportion‚0.9537528157234192 9"Purity/*FRt1SRqqt2FBt/SRqqt_purity‚0.5462963581085205 6 Purity/1FRt*SRqqt1FBt/FBt_purity‚0.994339644908905 7 Purity/1FRt2SRqqt1FBt/FRt_purity‚0.3499999940395355 ;"Purity/*FRt2SRqqt*FBt/event_purity‚0.007969309575855732 9"Purity/0FRt*SRqqt1FBt/event_purity‚0.7915863990783691 7 Purity/0FRt0SRqqt2FBt/FBt_purity‚0.8880225419998169 9"Purity/1FRt0SRqqt2FBt/event_purity‚0.4514928162097931 >&Purity/2FRt0SRqqt0FBt/event_proportion‚0.01990327425301075 *jet/accuracy_1_of_4‚0.6960964798927307 =&Purity/*FRt0SRqqt2FBt/event_proportion‚0.0103004090487957 :"Purity/0FRt1SRqqt*FBt/SRqqt_purity‚0.15404582023620605 & Purity/0FRt2SRqqt*FBt/FBt_purity‚1 @&Purity/0FRt2SRqqt1FBt/event_proportion‚0.0007207961170934141 ("Purity/1FRt1SRqqt2FBt/event_purity‚0 @&Purity/2FRt0SRqqt1FBt/event_proportion‚0.0007672990905120969 :"Purity/*FRt1SRqqt0FBt/SRqqt_purity‚0.12527161836624146 =&Purity/0FRt1SRqqt0FBt/event_proportion‚0.2083565890789032 ;"Purity/1FRt*SRqqt*FBt/event_purity‚0.059962522238492966 A&Purity/1FRt2SRqqt1FBt/event_proportion‚0.00027901786961592734 9"Purity/0FRt*SRqqt2FBt/SRqqt_purity‚0.6538461446762085 9"Purity/*FRt0SRqqt2FBt/event_purity‚0.6854254007339478 7 Purity/*FRt1SRqqt2FBt/FBt_purity‚0.5740740299224854b¢rÔ‚(:ç ÞîòÒ㣂¥ disk.dm-0.in‚ 1982.2109375 disk.dm-1.in‚0$ memory_percent‚99.20382499694824) proc.memory.percent‚17.25447177886963 network.sent‚ 1.677573e+06) disk./.usagePercent‚76.41045775830484" disk.nvme0c0n1.in‚ 1852.4453125& disk.nvme0n1.out‚15344.62646484375% disk.nvme1n1.out‚11008.5537109375 cpu‚25.79090559001414$ disk./.usageGB‚1173.091552734375 disk.nvme0n1.in‚ 1852.4453125 disk.nvme1c1n1.in‚ 129.765625' disk.nvme1c1n1.out‚11008.5537109375 disk.nvme1n1.in‚ 129.765625# disk.dm-0.out‚25455.52783203125 disk.rbd0.out‚0' proc.memory.availableMB‚ 130.4453125 network.recv‚263578( disk.nvme0c0n1.out‚15344.62646484375 disk.md0.out‚ 21.80859375 disk.dm-1.out‚ 850.892578125 disk.rbd0.in‚0# proc.memory.rssMB‚ 2826.97265625 proc.cpu.threads‚17 disk.md0.in‚0¬ú“Õ‚(:ˆ ÞîòÒþîç¤ gpu.0.gpu‚91.0 gpu.0.memory‚17, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚49.0 gpu.0.powerWatts‚115.272' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚38.424 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0 }škÖ‚(:à íîòÒèÙؤ network.sent‚ 1.735628e+06$ disk./.usageGB‚1173.092185974121" disk.nvme0c0n1.in‚ 1852.4453125$ disk.nvme1c1n1.out‚ 11073.1328125) proc.memory.percent‚17.25454330444336 cpu‚25.787570815548598 disk.rbd0.in‚0 disk.md0.in‚0 disk.dm-0.in‚ 1982.2109375# disk.dm-0.out‚25456.63330078125 disk.dm-1.in‚0# memory_percent‚99.2011308670044! proc.memory.rssMB‚ 2826.984375 proc.cpu.threads‚17) disk./.usagePercent‚76.41049900499078 disk.rbd0.out‚0 disk.nvme0n1.in‚ 1852.4453125 disk.nvme1n1.in‚ 129.765625 disk.dm-1.out‚ 914.923828125( proc.memory.availableMB‚ 130.88671875 network.recv‚267541( disk.nvme0c0n1.out‚15346.04541015625& disk.nvme0n1.out‚15346.04541015625 disk.nvme1c1n1.in‚ 129.765625" disk.nvme1n1.out‚ 11073.1328125 disk.md0.out‚ 22.21484375;1Éìׂ(:’ íîòÒí߇¥ gpu.0.gpu‚46.0 gpu.0.memory‚9, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚50.0 gpu.0.powerWatts‚189.166' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚63.05533333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0àÚ²0Ø+ _wandbruntime‚854  _runtime‚854Í€ÑÇÙÁ ' lr-AdamW/pg1‚0.00013859894801286795 ' lr-AdamW/pg2‚0.00013859894801286795 trainer/global_step‚1249 % _timestamp‚1.7844611802589617e+09 _runtime‚ 854.647386861 _step‚99t<øËÚÅ ' lr-AdamW/pg1‚0.00013859894801286795 ' lr-AdamW/pg2‚0.00013859894801286795 trainer/global_step‚1249 % _timestamp‚1.7844611802589617e+09 _runtime‚ 854.647386861 _step‚99cOÇ 0Û+ _wandbruntime‚854  _runtime‚854¯ôHxÜÁO RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 208873472 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 G?DeviceStatsMonitor.on_train_batch_CV±§O&start/segment.large_pool.peak‚682 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚12565952 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚9905621 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚9905621 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚9905623 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚12576095 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚73689454097408 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚96829849682330 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1061846281728 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1027957999268 _runtime‚ 854.653840441 trainer/global_step‚1249 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1030786939904 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1030698279936 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚6646066 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚2836853 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚96829953508116 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚97976517944832 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚9482919 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚95801995508848 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 _step‚100 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚9482751 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚72627607815680 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚96945731004928 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚97976412245504 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚96945713965568 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2836689 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372523008 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚96945731004928 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1030698279936 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚12565952 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚95801978469488 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚22481718 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚97976412245504 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1030786939904 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚72627513705984 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚9905623 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1027871212842 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚6646062 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1061731517952 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚12576095 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚97976517944832 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚22471573 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚164 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚168 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚96945713965568 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 % _timestamp‚1.7844611802620573e+09‘JTàË'ÝÅO M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1027871212842 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚6646062 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1061731517952 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚12576095 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 XEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚97976517944832 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚22471573 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚164 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚168 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 XEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚96945713965568 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 % _timestamp‚1.7844611802620573e+09 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 208873472 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚12565952 K?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚9905621 OCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚9905621 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 SGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚9905623 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚12576095 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚73689454097408 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 TADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚96829849682330 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1061846281728 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1027957999268 _runtime‚ 854.653840441 trainer/global_step‚1249 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1030786939904 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1030698279936 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚6646066 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚2836853 XEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚96829953508116 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 UBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚97976517944832 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚9482919 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚95801995508848 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 _step‚100 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚9482751 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚72627607815680 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 _LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚96945731004928 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 TADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚97976412245504 [HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚96945713965568 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2836689 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372523008 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 \IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚96945731004928 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1030698279936 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚12565952 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚95801978469488 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚22481718 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 Q>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚97976412245504 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1030786939904 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚72627513705984 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 OCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚9905623d,§gÞ‚(:Ü üîòÒ—Ëð¤# disk.dm-0.out‚25463.83349609375( proc.memory.availableMB‚ 129.51953125 disk.nvme1n1.in‚ 129.765625 disk.nvme1n1.out‚ 11075.34375 disk.md0.out‚ 22.52734375$ memory_percent‚99.20947551727295 proc.memory.rssMB‚ 2827.5625* proc.memory.percent‚17.258071899414062 cpu‚25.79151617244749 network.sent‚ 1.738651e+06% disk./.usageGB‚1173.0899620056152( disk.nvme0c0n1.out‚15351.80810546875& disk.nvme0n1.out‚15351.80810546875 disk.nvme1c1n1.in‚ 129.765625 disk.dm-0.in‚ 1982.2109375 proc.cpu.threads‚17 network.recv‚271198 disk.dm-1.in‚0" disk.nvme1c1n1.out‚ 11075.34375 disk.md0.in‚0) disk./.usagePercent‚76.41035414464193 disk.dm-1.out‚ 915.009765625 disk.rbd0.in‚0 disk.rbd0.out‚0" disk.nvme0c0n1.in‚ 1852.4453125 disk.nvme0n1.in‚ 1852.4453125ðSôžß‚(:“ üîòÒÚ£¦¦ gpu.0.gpu‚91.0 gpu.0.memory‚16, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚50.0 gpu.0.powerWatts‚226.681' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚75.56033333333335 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0»Æ.Œ0à+ _wandbruntime‚855  _runtime‚855í‚*eÐ&áÊM SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2839076 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1062654042112 trainer/global_step‚1249 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚97020880671232 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚73746399585792 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚9490018 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚180 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚9913250 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1031572662784 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2838900 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 _runtime‚ 855.057554941 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚98052453334016 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚9490198 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚6651122 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚6651118 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 206769152 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372523008 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 112659456 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 _step‚101 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚176 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚22499181 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚72683745543680 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚9913252 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚22489022 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1031572662784 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚72683651433984 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚97020880671232 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 $ _timestamp‚1.784461180669108e+09 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1028741493980 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚22499181 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚95876286034240 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚98052347627520 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚22489022 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚96904923702322 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚9913252 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1031483995648 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚98052453334016 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496ž¶Å°âÎM MADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚9913250 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚12585929 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 $ _timestamp‚1.784461180669108e+09 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1028741493980 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚22499181 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚95876286034240 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 R?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚98052347627520 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚22489022 R?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚96904923702322 MADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚9913252 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1031483995648 VCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚98052453334016 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2839076 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1062654042112 trainer/global_step‚1249 ZGDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚97020880671232 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚73746399585792 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚9490018 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚180 I=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚9913250 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1031572662784 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚2838900 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 _runtime‚ 855.057554941 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 S@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚98052453334016 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚9490198 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚6651122 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚6651118 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 206769152 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372523008 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 112659456 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 _step‚101 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚176 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚22499181 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚72683745543680 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 QEDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚9913252 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚22489022 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1031572662784 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚72683651433984 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 ]JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚97020880671232 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 EDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚100703197280768 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚10287071 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚76536994630144 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚100703214320128 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1070136880128 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2946205 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1070048220160 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚10287071 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚13068947 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 trainer/global_step‚1299 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚101773245500928 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚100703197280768 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1067196530076 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 _runtime‚ 887.283817282 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚13058804 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚99516521778448 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚23356020 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1070048220160 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚75434400105984 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚10287073 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚13058804 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚9845255 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚9845067 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚188 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1102381456384 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚75434494215680 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 118958080 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚6898866 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚101773351200256 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚101773245500928 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚99516504739088 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚13068947 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 _step‚104 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚100583614482738 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚100703197280768 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚10287071 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚76536994630144 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚100703214320128 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1070136880128 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚2946205 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1070048220160 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚10287071 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚13068947 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 trainer/global_step‚1299 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚101773245500928 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚100703197280768 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1067196530076 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 _runtime‚ 887.283817282 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚13058804 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 _LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚99516521778448 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚23356020 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1070048220160 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚75434400105984 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚10287073 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚13058804 PDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚9845255 L@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚9845067 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚188 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1102381456384 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚75434494215680 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 118958080 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚6898866 TDDeviceStatsMonitor.on_train_bat§ £¾bch_start/reserved_bytes.all.allocated‚ 14638120960 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚101773351200256 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚101773245500928 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 [HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚99516504739088 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚13068947h“pN40ð+ _wandbruntime‚887  _runtime‚887DáGß&ñÙM [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 127339520 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚10294700 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚23363324 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚9852469 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚6903918 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚99590812303840 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1070922603008 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚101849286589440 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚101849286589440 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1070833935872 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚99590795264480 % _timestamp‚1.7844612133094032e+09 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚23373483 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚13068624 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚100778363986432 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚6903922 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚9852282 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚183 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚23363324 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚101849180882944 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚23373483 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚13078781 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚100778346947072 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 _step‚105 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚100778363986432 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚10294702 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚100778346947072 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1070922603008 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1067893238250 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 221449216 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚100658688502730 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1067980024788 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2948547 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚75490631943680 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1103321806336 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 _step‚105 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚100778363986432 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚10294702 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚100778346947072 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1070922603008 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1067893238250 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 221449216 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚100658688502730 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1067980024788 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚2948547 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚75490631943680 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1103321806336 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚23363324 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 NBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚9852469 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚6903918 ]JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚99590812303840 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1070922603008 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚101849286589440 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚101849286589440 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1070833935872 YFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚99590795264480 % _timestamp‚1.7844612133094032e+09 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚23373483 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚13068624 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚100778363986432 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚6903922 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 J>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚9852282 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚183 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚23363324 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚101849180882944 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚23373483 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚13078781 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚100778346947072 FDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚105570078756352 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚13551656 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚104460680595968 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1109486820352 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚3053920 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 % _timestamp‚1.7844612457056751e+09 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚24220177 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚10668521 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚104337379283146 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚79384447109632 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚78241286505984 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1106435060884 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚104460697635328 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1109486820352 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚104460697635328 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1142939147264 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚10668521’Ô2+Ü'ÿÖO C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚13561799 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚104460680595968 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1109486820352 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚3053920 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 % _timestamp‚1.7844612457056751e+09 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚24220177 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚10668521 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚104337379283146 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚79384447109632 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚78241286505984 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1106435060884 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚104460697635328 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1109486820352 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚104460697635328 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1142939147264 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚10668521 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚105570078756352 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚13551656 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚104412453303138 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1110272543232 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 217254912 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚104535830262272 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚174 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚10212842 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚10676150 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚103305338573440 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚103305321534080 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1110183876096 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚79441390345216 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 _runtime‚ 920.499746122 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚10676152 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚24247785 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚178 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 _step‚109 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1143872001536 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1110183876096 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3056124 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚7156722 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚3056298 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1107218555596 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚104535830262272 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚104535847301632 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372523008 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚105646119844864 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 trainer/global_step‚1349 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚10213020 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚104412557129036 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1110272543232 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 123145216 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚10676150 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚13571633 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚78297518343680 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚24237626 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚104535847301632 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚79441173090304 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682ôJH%%‚áM ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1110183876096 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3056124 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚7156722 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚3056298 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1107218555596 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚104535830262272 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚104535847301632 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372523008 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚105646119844864 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 trainer/global_step‚1349 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚10213020 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚104412557129036 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1110272543232 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 123145216 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚10676150 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚13571633 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚78297518343680 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚24237626 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚104535847301632 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚79441173090304 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚13561476 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚104412453303138 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1110272543232 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 217254912 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚104535830262272 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚174 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚10212842 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚10676150 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚103305338573440 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚103305321534080 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1110183876096 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚79441390345216 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 _runtime‚ 920.499746122 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚10676152 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚24247785 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚178 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 _step‚109 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1143872001536 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130m™&c0ƒ+ _wandbruntime‚920  _runtime‚920Ý˵„ý 0loss/FRt2/assignment_loss‚2.0176456622372103 /loss/FRt2/detection_loss‚1.4246557025854212 /loss/FBt1/detection_loss‚2.4948644891781813 2loss/SRqqt1/assignment_loss‚1.4145682379310824 1loss/FBt1/assignment_loss‚0.15048500984072097 1loss/SRqqt2/detection_loss‚0.8347145850532853 /loss/FBt2/detection_loss‚2.4333719752025686 &loss/total_loss‚18.814532804902882 trainer/global_step‚1349 /loss/FRt1/assignment_loss‚3.430953879537605 1loss/SRqqt2/assignment_loss‚1.105988083754566 1loss/FBt2/assignment_loss‚0.11861301190912005 /loss/FRt1/detection_loss‚0.5453199671780206 epoch‚3 $ _timestamp‚1.784461246112169e+09 _runtime‚ 920.500238513 _step‚110 0loss/SRqqt1/detection_loss‚2.843352200495101Œ¼›8… 1loss/FBt1/assignment_loss‚0.15048500984072097 1loss/SRqqt2/detection_loss‚0.8347145850532853 /loss/FBt2/detection_loss‚2.4333719752025686 &loss/total_loss‚18.814532804902882 trainer/global_step‚1349 /loss/FRt1/assignment_loss‚3.430953879537605 1loss/SRqqt2/assignment_loss‚1.105988083754566 1loss/FBt2/assignment_loss‚0.11861301190912005 /loss/FRt1/detection_loss‚0.5453199671780206 epoch‚3 $ _timestamp‚1.784461246112169e+09 _runtime‚ 920.500238513 _step‚110 0loss/SRqqt1/detection_loss‚2.843352200495101 0loss/FRt2/assignment_loss‚2.0176456622372103 /loss/FRt2/detection_loss‚1.4246557025854212 /loss/FBt1/detection_loss‚2.4948644891781813 2loss/SRqqt1/assignment_loss‚1.4145682379310824nÆn†‚(:ã ÇïòÒ—öž¥ network.sent‚ 1.926657e+06 disk.nvme0n1.in‚ 1852.4453125 disk.nvme1c1n1.in‚ 129.765625 disk.md0.in‚0 disk.dm-0.in‚ 1982.2109375 disk.dm-1.in‚0 disk.rbd0.out‚0 proc.cpu.threads‚17( disk.nvme0c0n1.out‚15558.47705078125& disk.nvme0n1.out‚15558.47705078125 disk.nvme1n1.in‚ 129.765625% disk.nvme1n1.out‚11132.9658203125$ memory_percent‚99.20845031738281$ proc.memory.availableMB‚129.6875 proc.memory.rssMB‚ 2827.5625) disk./.usagePercent‚76.42583208778281" disk.nvme0c0n1.in‚ 1852.4453125' disk.nvme1c1n1.out‚11132.9658203125# disk.dm-0.out‚25723.11474609375 disk.dm-1.out‚ 915.681640625 disk.rbd0.in‚0* proc.memory.percent‚17.258071899414062 cpu‚25.798794321141628 network.recv‚289859% disk./.usageGB‚1173.3275871276855 disk.md0.out‚ 24.55859375.VC‘‡‚(:† ÇïòÒÕÓÐ¥ gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚48.0 gpu.0.powerWatts‚114.231' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚38.077 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¿$lˆ‚(:á ÖïòÒîÑᤠnetwork.sent‚ 1.978176e+06) disk./.usagePercent‚76.42591135099256 disk.dm-0.out‚ 25727.0625 disk.rbd0.in‚0 disk.rbd0.out‚0& disk.nvme0n1.out‚15562.09814453125( disk.nvme1c1n1.out‚11134.06494140625 network.recv‚294120( disk.nvme0c0n1.out‚15562.09814453125 disk.nvme0n1.in‚ 1852.4453125* proc.memory.percent‚17.258071899414062% disk./.usageGB‚1173.3288040161133 disk.md0.in‚0 disk.md0.out‚ 24.89453125 disk.dm-1.out‚ 915.732421875" disk.nvme0c0n1.in‚ 1852.4453125 disk.nvme1n1.in‚ 129.765625& disk.nvme1n1.out‚11134.06494140625$ memory_percent‚99.20744895935059 disk.dm-0.in‚ 1982.2109375 disk.dm-1.in‚0 disk.nvme1c1n1.in‚ 129.765625' proc.memory.availableMB‚ 129.8515625 proc.memory.rssMB‚ 2827.5625 cpu‚25.799883844434454 proc.cpu.threads‚17zË5Õž‰‚(:“ ÖïòÒÖ•ü¦ gpu.0.gpu‚47.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚48.0 gpu.0.powerWatts‚117.019' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚39.00633333333334 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ˆ"/c0Š+ _wandbruntime‚953  _runtime‚953œˆs0È‹Â ' lr-AdamW/pg1‚0.00013835194990674958 ' lr-AdamW/pg2‚0.00013835194990674958 trainer/global_step‚1399 % _timestamp‚1.7844612789222558e+09 _runtime‚ 953.310655187 _step‚1114‡)áÌŒÆ _runtime‚ 953.310655187 _step‚111 ' lr-AdamW/pg1‚0.00013835194990674958 ' lr-AdamW/pg2‚0.00013835194990674958 trainer/global_step‚1399 % _timestamp‚1.7844612789222558e+09oò1¦0+ _wandbruntime‚953  _runtime‚953´FÙ'ŽÓO JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚185 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚14044508 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚108218163911168 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚11049973 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚14044508 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚10566369 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1148836760576 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚108218180950528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1148748100608 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚82231676674560 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1145586805266 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1145673591692 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚14054651 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚81048172905984 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚109366912011776 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚10566184 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372523008 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚106945574317648 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1148836760576 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚108218180950528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1148748100608 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚82231676674560 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1145586805266 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1145673591692 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚14054651 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚81048172905984 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚109366912011776 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚10566184 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372523008 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚106945574317648 IDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚108293330616832 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372523008 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚ 953.736555213 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚108293313577472 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1149622483456 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1149533816320 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1146457086404 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚14064485 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚7409518 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚14054328 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚25122087 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚108293330616832 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚177 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚81104404743680 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚25122087 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚108166218103546 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚25111928 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 217254912 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚10573452 % _timestamp‚1.7844612793479333e+09 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚109442847393792 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1149622483456 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚10573629 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚7409522 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1184318474752 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚107019864843040 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚108293313577472 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚108166321929444 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1146370299866 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 _step‚113 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130Xç&’áM G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚25122087 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚108293330616832 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚177 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚81104404743680 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚25122087 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚108166218103546 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚25111928 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 217254912 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚10573452 % _timestamp‚1.7844612793479333e+09 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚109442847393792 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1149622483456 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚10573629 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚7409522 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1184318474752 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚107019864843040 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚108293313577472 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚108166321929444 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1146370299866 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 _step‚113 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚11057600 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚108293330616832 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372523008 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚ 953.736555213 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚108293313577472 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1149622483456 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1149533816320 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1146457086404 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚14064485 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚7409518 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚14054328 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360qR×YÖ0“+ _wandbruntime‚953  _runtime‚953@- ”ý 0loss/FBt1/assignment_loss‚0.1843947643520751 /loss/FRt1/detection_loss‚0.5769998781528736 0loss/SRqqt1/detection_loss‚2.405261132961177 0loss/SRqqt2/detection_loss‚0.742174485494437 &loss/total_loss‚17.633012761825615 0loss/FRt1/assignment_loss‚3.3447776432417653 /loss/FBt1/detection_loss‚1.8365444242755098 .loss/FBt2/detection_loss‚2.594897720581298 0loss/FRt2/assignment_loss‚1.8192275366451167 2loss/SRqqt1/assignment_loss‚1.5141621779042238 2loss/SRqqt2/assignment_loss‚1.0465128786844755 /loss/FRt2/detection_loss‚1.3645214205155227 trainer/global_step‚1399 _runtime‚ 953.737153614 1loss/FBt2/assignment_loss‚0.20353869901714014 epoch‚3 % _timestamp‚1.7844612793490796e+09 _step‚114-&³’• 0loss/FBt1/assignment_loss‚0.1843947643520751 /loss/FRt1/detection_loss‚0.5769998781528736 0loss/SRqqt1/detection_loss‚2.405261132961177 0loss/SRqqt2/detection_loss‚0.742174485494437 &loss/total_loss‚17.633012761825615 0loss/FRt1/assignment_loss‚3.3447776432417653 /loss/FBt1/detection_loss‚1.8365444242755098 .loss/FBt2/detection_loss‚2.594897720581298 0loss/FRt2/assignment_loss‚1.8192275366451167 2loss/SRqqt1/assignment_loss‚1.5141621779042238 2loss/SRqqt2/assignment_loss‚1.0465128786844755 /loss/FRt2/detection_loss‚1.3645214205155227 trainer/global_step‚1399 _runtime‚ 953.737153614 1loss/FBt2/assignment_loss‚0.20353869901714014 epoch‚3 % _timestamp‚1.7844612793490796e+09 _step‚114rQŽAwk–‚(:à åïòÒ³¦Ó¤! disk.dm-1.out‚915.86376953125* proc.memory.percent‚17.258071899414062 proc.cpu.threads‚17 disk.rbd0.in‚0& disk.nvme0c0n1.out‚15563.001953125 disk.nvme1c1n1.in‚ 129.765625& disk.nvme1n1.out‚11134.72119140625 cpu‚25.796296295414177 network.recv‚297167 disk.nvme0n1.in‚ 1852.4453125 disk.md0.in‚0( proc.memory.availableMB‚ 129.81640625 proc.memory.rssMB‚ 2827.5625 disk.rbd0.out‚0" disk.nvme0c0n1.in‚ 1852.4453125$ disk.nvme0n1.out‚15563.001953125( disk.nvme1c1n1.out‚11134.72119140625 disk.md0.out‚ 25.265625 disk.dm-0.out‚25727.70703125 disk.dm-1.in‚0$ memory_percent‚99.20766353607178 network.sent‚ 1.98114e+06) disk./.usagePercent‚76.24888529588843$ disk./.usageGB‚1170.611011505127 disk.nvme1n1.in‚ 129.765625 disk.dm-0.in‚ 1982.2109375&D¦l“—‚(:ˆ åïòÒãó´¥ gpu.0.gpu‚91.0 gpu.0.memory‚18, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚49.0 gpu.0.powerWatts‚218.391' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚72.797 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0æq½Èj˜‚(:ß ôïòÒ‰ÞÕ¤$ memory_percent‚99.20461177825928 proc.cpu.threads‚17 network.sent‚ 2.044709e+06% disk./.usageGB‚1169.2741584777832 disk.nvme1c1n1.in‚ 129.765625 disk.md0.in‚0" disk.dm-0.out‚25731.7255859375( proc.memory.availableMB‚ 130.31640625 network.recv‚301462) disk./.usagePercent‚76.16180807541333( disk.nvme1c1n1.out‚11136.04833984375 disk.nvme1n1.in‚ 129.765625 disk.md0.out‚ 25.8515625 disk.dm-0.in‚ 1982.2109375 disk.rbd0.out‚0" disk.nvme0c0n1.in‚ 1852.4453125 disk.nvme0n1.in‚ 1852.4453125 disk.dm-1.in‚0! disk.dm-1.out‚915.93798828125 disk.rbd0.in‚0# disk.nvme0c0n1.out‚ 15567.015625! proc.memory.rssMB‚ 2826.984375) proc.memory.percent‚17.25454330444336 cpu‚25.79281051503714! disk.nvme0n1.out‚ 15567.015625& disk.nvme1n1.out‚11136.04833984375û)‘™‚(:† ôïòÒ§³Ÿ¥ gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚48.0 gpu.0.powerWatts‚114.231' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚38.077 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0†:ð¯0š+ _wandbruntime‚987  _runtime‚987“ŒRÝț _step‚115 ' lr-AdamW/pg1‚0.00013826078098687992 ' lr-AdamW/pg2‚0.00013826078098687992 trainer/global_step‚1449 % _timestamp‚1.7844613127419384e+09 _runtime‚ 987.131052859¾Ç‘ ÌœÆ trainer/global_step‚1449 % _timestamp‚1.7844613127419384e+09 _runtime‚ 987.131052859 _step‚115 ' lr-AdamW/pg1‚0.00013826078098687992 ' lr-AdamW/pg2‚0.00013826078098687992sü îj0+ _wandbruntime‚987  _runtime‚987{×Ù'žÓO YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚113163850966528 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚10926588 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 trainer/global_step‚1449 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚14547503 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1224054514176 _runtime‚ 987.131319429 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚7657262 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚85079333179392 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚11431423 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚173 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚11431421 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 125249536 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚11431421 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚113163745267200 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚111975647226368 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚14537360 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚11431423 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚83855153415680 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚110660100587248 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚111844908883962 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚25968781 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 % _timestamp‚1.7844613127427878e+09 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚113163745267200 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚177 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚85079113820160 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚25978926 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚3269322 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1184912122500 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚110660083547888 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚3269149 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚83855059305984 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 _step‚116 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1224179763712 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1184825336074 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚14547503 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚113163850966528 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚111975664265728 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚10926411 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1188186700800 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1188098040832 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 219359232 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚10926411 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1188186700800 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1188098040832 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 219359232 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚11431423 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚173 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚11431421 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 125249536 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚11431421 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚113163745267200 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚111975647226368 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚14537360 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚11431423 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚83855153415680 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚110660100587248 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚111844908883962 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚25968781 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 % _timestamp‚1.7844613127427878e+09 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚113163745267200 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚177 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚85079113820160 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚25978926 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚3269322 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1184912122500 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚110660083547888 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚3269149 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚83855059305984 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0tÙ€Fã0 + _wandbruntime‚987  _runtime‚987Ñ’F'ã&¡ÝM ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1185695617212 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚25986230 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚85136062461440 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚110734391112640 trainer/global_step‚1449 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚10933622 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚83911291143680 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚170 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚10933796 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚112050813932032 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚11439052 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚113239786355712 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3271304 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚11439050 _step‚117 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1188883756544 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚25996389 % _timestamp‚1.7844613131674333e+09 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1185608830674 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚112050813932032 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚174 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚113239786355712 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1188972423680 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1188972423680 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚14557337 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130þ’ ç&¢áM ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚112050796892672 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372523008 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1185695617212 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚25986230 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚85136062461440 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚110734391112640 trainer/global_step‚1449 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚10933622 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚83911291143680 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚170 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚10933796 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚112050813932032 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚11439052 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚113239786355712 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3271304 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚11439050 _step‚117 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1188883756544 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚25996389 % _timestamp‚1.7844613131674333e+09 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1185608830674 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚112050813932032 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚174 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚113239786355712 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1188972423680 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1188972423680 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚14557337 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 B ì樀 & lr-AdamW/pg2‚0.0001381652034811084 trainer/global_step‚1499 % _timestamp‚1.7844613468463795e+09 _runtime‚ 1021.23480455 _step‚119 & lr-AdamW/pg1‚0.0001381652034811084wõôÁÂ2¯- _wandbruntime‚1021  _runtime‚1021‰“Œ[°ÔO WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚3376783 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1224063866882 _runtime‚1021.235833412 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1224150653308 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚7910066 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚185 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚115598777510156 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚115733130541568 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peaYâ'}"k‚682 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚116960684221952 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚11812873 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1227536641024 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 231942144 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚115733147580928 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1227447981056 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚11812871 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚3376598 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚116960578522624 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚116960684221952 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚11812871 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1227536641024 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚87926734708224 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚86662039815680 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 % _timestamp‚1.7844613468474581e+09 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 _step‚120 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚11812873 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚15030212 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1227447981056 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚7910062 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚114374626856848 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 trainer/global_step‚1499e·ŽÞ'±ØO @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚116960578522624 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚116960684221952 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚11812871 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1227536641024 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚87926734708224 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚86662039815680 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 % _timestamp‚1.7844613468474581e+09 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 _step‚120 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚11812873 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚15030212 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1227447981056 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚7910062 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚114374626856848 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 trainer/global_step‚1499 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚3376783 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1224063866882 _runtime‚1021.235833412 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1224150653308 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚7910066 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚185 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚115598777510156 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚115733130541568 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚116960684221952 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚11812873 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1227536641024 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 231942144 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚115733147580928 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1227447981056 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚11812871 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚3376598x Ü2$2²- _wandbruntime‚1021  _runtime‚1021“Á)ã&³ÝM =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚86718083433984 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1265366597120 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚115808280207872 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1228322363904 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚87983673577472 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚114448900342880 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚26870691 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1228233696768 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚3378903 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚115808297247232 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚117036513904640 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚11820502 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚26860532 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1228322363904 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚115808280207872 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚11294025 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚15040032 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚11820502 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚15050189 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚11293834 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚7915122 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚86718177543680 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚115808297247232 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3378716 _step‚121 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚15050189 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1224847361482 _runtime‚1021.657938382 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚26860532 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚26870691 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 129436672 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 $ _timestamp‚1.784461347269384e+09 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚7915118 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚114448917382240 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1265496033792 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚115673747704362 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812¾_§Ò´áM ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚86718083433984 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1265366597120 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚115808280207872 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1228322363904 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚87983673577472 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚114448900342880 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚26870691 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1228233696768 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚3378903 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚115808297247232 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 C;Deviäæ’ûceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚117036513904640 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚11820502 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚26860532 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1228322363904 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚115808280207872 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚11294025 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚15040032 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚11820502 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚15050189 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚11293834 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚7915122 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚86718177543680 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚115808297247232 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3378716 _step‚121 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚15050189 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1224847361482 _runtime‚1021.657938382 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚26860532 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚26870691 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 129436672 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 $ _timestamp‚1.784461347269384e+09 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚7915118 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚114448917382240 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1265496033792 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚115673747704362 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 P2µ- _wandbruntime‚1021  _runtime‚1021Äoåm¶ý _step‚122 2loss/SRqqt1/assignment_loss‚1.2031271553941463 1loss/SRqqt1/detection_loss‚2.1230561150751064 &loss/total_loss‚19.301511652109618 _runtime‚1021.658736293 /loss/FRt1/assignment_loss‚3.191317044804037 2loss/SRqqt2/assignment_loss‚1.0542053857581182 1loss/FBt1/assignment_loss‚0.16745785795477827 0loss/FBt2/assignment_loss‚0.3420463918775726 /loss/FRt1/detection_loss‚0.6157837649797735 % _timestamp‚1.7844613472704785e+09 0loss/SRqqt2/detection_loss‚0.786790215512667 .loss/FBt1/detection_loss‚2.261475402040385 epoch‚3 trainer/global_step‚1499 0loss/FRt2/assignment_loss‚1.7205281031187887 /loss/FRt2/detection_loss‚1.3948412180356908 .loss/FBt2/detection_loss‚4.440882997558556tm<®· /loss/FRt1/assignment_loss‚3.191317044804037 2loss/SRqqt2/assignment_loss‚1.0542053857581182 1loss/FBt1/assignment_loss‚0.16745785795477827 0loss/FBt2/assignment_loss‚0.3420463918775726 /loss/FRt1/detection_loss‚0.6157837649797735 % _timestamp‚1.7844613472704785e+09 0loss/SRqqt2/detection_loss‚0.786790215512667 .loss/FBt1/detection_loss‚2.261475402040385 epoch‚3 trainer/global_step‚1499 0loss/FRt2/assignment_loss‚1.7205281031187887 /loss/FRt2/detection_loss‚1.3948412180356908 .loss/FBt2/detection_loss‚4.440882997558556 _step‚122 2loss/SRqqt1/assignment_loss‚1.2031271553941463 1loss/SRqqt1/detection_loss‚2.1230561150751064 &loss/total_loss‚19.301511652109618 _runtime‚1021.658736293z÷©a$l¸‚(:á °ðòÒ©¼Á¤ network.recv‚315299) disk./.usagePercent‚76.34168561827649% disk./.usageGB‚1172.0357284545898" disk.nvme1n1.out‚ 11140.4765625 disk.md0.in‚0" disk.nvme0c0n1.in‚ 1852.4453125 network.sent‚ 2.121764e+06 disk.dm-0.in‚ 1982.2109375 disk.dm-1.in‚0 disk.dm-1.out‚916.4365234375 disk.nvme1c1n1.in‚ 129.765625 cpu‚25.779400190031755 proc.cpu.threads‚17 disk.nvme1n1.in‚ 129.765625# disk.dm-0.out‚27660.53662109375 disk.rbd0.in‚0 disk.rbd0.out‚0( disk.nvme0c0n1.out‚17495.77197265625& disk.nvme0n1.out‚17495.77197265625 proc.memory.rssMB‚ 2827.5625* proc.memory.percent‚17.258071899414062 disk.md0.out‚ 27.67578125 disk.nvme0n1.in‚ 1852.4453125$ disk.nvme1c1n1.out‚ 11140.4765625$ memory_percent‚99.21345710754395' proc.memory.availableMB‚ 128.8671875ëåËò¹‚(:’ °ðòÒ‘ûñ¤ gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚48.0 gpu.0.powerWatts‚134.511' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚44.836999999999996 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¥û@ânº‚(:ã ¿ðòÒæ–ʤ disk.nvme1c1n1.in‚ 129.765625 disk.nvme1n1.in‚ 129.765625 disk.md0.out‚28.0625# disk.dm-0.out‚28529.58837890625$ memory_percent‚99.60072040557861% disk./.usageGB‚1171.9894523620605 disk.nvme0n1.in‚ 1852.4453125' disk.nvme1c1n1.out‚11270.0048828125% disk.nvme1n1.out‚11270.0048828125 disk.md0.in‚0" proc.memory.rssMB‚ 2826.9765625 cpu‚25.775220688134954 proc.cpu.threads‚17 network.recv‚319355 disk.rbd0.out‚0 disk.dm-0.in‚ 1982.2109375 disk.dm-1.out‚ 916.51171875' proc.memory.availableMB‚ 65.41796875 disk.rbd0.in‚0" disk.nvme0c0n1.in‚ 1852.4453125( disk.nvme0c0n1.out‚18236.19873046875 disk.dm-1.in‚0) proc.memory.percent‚17.25449562072754 network.sent‚ 2.183256e+06) disk./.usagePercent‚76.33867138003977& disk.nvme0n1.out‚18236.19873046875É.ºž»‚(:“ ¿ðòÒ≥ gpu.0.gpu‚90.0 gpu.0.memory‚16, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚49.0 gpu.0.powerWatts‚221.501' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚73.83366666666666 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0#`Á2¼- _wandbruntime‚1055  _runtime‚1055Úåj_ɽà ' lr-AdamW/pg1‚0.00013806522352819952 ' lr-AdamW/pg2‚0.00013806522352819952 trainer/global_step‚1549 % _timestamp‚1.7844613808319423e+09 _runtime‚1055.220364695 _step‚123VÁdCÍ¾Ç trainer/global_step‚1549 % _timestamp‚1.7844613808319423e+09 _runtime‚1055.220364695 _step‚123 ' lr-AdamW/pg1‚0.00013806522352819952 ' lr-AdamW/pg2‚0.00013806522352819952{WX52¿- _wandbruntime‚1055  _runtime‚1055ïiøoÚ'ÀÔO O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚12194321 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚120757517477376 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚12194321 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚11647067 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚119490630896128 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚118089136087088 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚89468832105984 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚118089153126448 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1263302397690 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚8162866 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372523008 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1305104634880 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚119490613856768 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1266797921280 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚15523064 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 217262080 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 % _timestamp‚1.7844613808330665e+09 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚15533207 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚120757517477376 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚15533207 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚119352542310564 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚90774154002944 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚119490630896128 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚120757411778048 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚12194323 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1266886581248 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚89468926215680 trainer/global_step‚1549 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1266797921280 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚171 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚3484205 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 _runtime‚1055.222427838 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚3484372 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚119490613856768 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1263389184116 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚120757411778048 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚11647238 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚167 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚15523064 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚90773936740864 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚27717385 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚08¹;>Þ'ÁØO E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚27717385 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚12194321 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚120757517477376 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚12194321 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚11647067 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚119490630896128 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚118089136087088 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚89468832105984 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚118089153126448 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1263302397690 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚8162866 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372523008 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1305104634880 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚119490613856768 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1266797921280 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚15523064 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 217262080 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 % _timestamp‚1.7844613808330665e+09 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚15533207 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚120757517477376 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚15533207 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚119352542310564 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚90774154002944 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚119490630896128 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚120757411778048 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚12194323 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1266886581248 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚89468926215680 trainer/global_step‚1549 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1266797921280 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚171 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚3484205 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 _runtime‚1055.222427838 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚3484372 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚119490613856768 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1263389184116 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚120757411778048 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚11647238 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚167 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚15523064 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚90773936740864 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130|$úÉ62Â- _wandbruntime‚1055  _runtime‚1055¿_›FÃÞM \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1264172678828 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚120833347160064 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚11654228 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚15532884 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚119565763523072 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚681x× ž!2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1267583636992 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚8167922 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 127339520 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 _step‚125 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚27734834 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚119427616330668 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚12201952 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 221449216 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚164 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚12201952 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚27734834 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1306039455232 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 _runtime‚1055.643824783 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚15543041 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚90830881949696 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚119427512504770 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 % _timestamp‚1.7844613812553031e+09 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚27744993 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚120833452866560 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚118163426612480 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚12201950 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚118163443651840 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚27744993 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚90831103398912 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚120833452866560 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1305912115712 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1267672304128 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚12201950 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚3486470 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372523008 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚89525063943680 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1267583636992 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 trainer/global_step‚1549n)‡ßè&ÄâM BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚15532884 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚119565763523072 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1267583636992 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚8167922 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 127339520 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 _step‚125 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚27734834 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚119427616330668 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚12201952 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 221449216 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚164 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚12201952 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚27734834 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1306039455232 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 _runtime‚1055.643824783 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚15543041 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚90830881949696 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚119427512504770 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 % _timestamp‚1.7844613812553031e+09 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚27744993 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚120833452866560 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚118163426612480 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚12201950 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚118163443651840 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚27744993 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚90831103398912 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚120833452866560 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1305912115712 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1267672304128 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚12201950 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚3486470 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372523008 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚89525063943680 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1267583636992 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 trainer/global_step‚1549 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1264172678828 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚120833347160064 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚11654228}ß;£,2Å- _wandbruntime‚1055  _runtime‚1055³®Õ.Æÿ 2loss/SRqqt2/assignment_loss‚0.9887178688379277 1loss/FBt1/assignment_loss‚0.11835445674283795 trainer/global_step‚1549 _runtime‚1055.644553334 /loss/FRt1/assignment_loss‚3.270216142425702 0loss/FRt2/assignment_loss‚1.6533476150168103 1loss/FBt2/assignment_loss‚0.32698156427198766 % _timestamp‚1.7844613812563531e+09 /loss/FRt1/detection_loss‚0.6399754746278398 .loss/FRt2/detection_loss‚1.363498234782811 1loss/SRqqt2/detection_loss‚0.7450834638407887 /loss/FBt1/detection_loss‚2.5692244549969745 %loss/total_loss‚18.60022589446486 _step‚126 2loss/SRqqt1/assignment_loss‚1.2848036195115884 1loss/SRqqt1/detection_loss‚1.9215279490244193 /loss/FBt2/detection_loss‚3.7184950503851724 epoch‚3ÑÈË¢ ǃ /loss/FBt1/detection_loss‚2.5692244549969745 %loss/total_loss‚18.60022589446486 _step‚126 2loss/SRqqt1/assignment_loss‚1.2848036195115884 1loss/SRqqt1/detection_loss‚1.9215279490244193 /loss/FBt2/detection_loss‚3.7184950503851724 epoch‚3 2loss/SRqqt2/assignment_loss‚0.9887178688379277 1loss/FBt1/assignment_loss‚0.11835445674283795 trainer/global_step‚1549 _runtime‚1055.644553334 /loss/FRt1/assignment_loss‚3.270216142425702 0loss/FRt2/assignment_loss‚1.6533476150168103 1loss/FBt2/assignment_loss‚0.32698156427198766 % _timestamp‚1.7844613812563531e+09 /loss/FRt1/detection_loss‚0.6399754746278398 .loss/FRt2/detection_loss‚1.363498234782811 1loss/SRqqt2/detection_loss‚0.7450834638407887~Š%¦äkÈ‚(:à ÎðòÒø£ð¤ network.sent‚ 2.188119e+06 disk.rbd0.in‚0& disk.nvme1c1n1.out‚11271.646484375 disk.nvme1n1.in‚ 129.765625$ proc.memory.availableMB‚64.59375 proc.cpu.threads‚17) disk./.usagePercent‚76.33870790571949 disk.md0.out‚ 28.37890625 disk.dm-0.in‚ 1982.2109375 disk.dm-1.in‚0( disk.nvme0c0n1.out‚18244.14306640625 disk.nvme0n1.in‚ 1852.4453125& disk.nvme0n1.out‚18244.14306640625 disk.nvme1c1n1.in‚ 129.765625 network.recv‚322838 disk.md0.in‚0 disk.rbd0.out‚0" disk.nvme0c0n1.in‚ 1852.4453125$ disk.nvme1n1.out‚11271.646484375$ memory_percent‚99.60575103759766 proc.memory.rssMB‚ 2827.5625 cpu‚25.776477729026183% disk./.usageGB‚1171.9900131225586# disk.dm-0.out‚28538.42236328125 disk.dm-1.out‚ 916.58203125* proc.memory.percent‚17.258071899414062ž ø“É‚(:ˆ ÎðòÒ««¥ gpu.0.gpu‚80.0 gpu.0.memory‚19, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚48.0 gpu.0.powerWatts‚197.619' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚65.873 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0fÑŸÊ‚(:” ÝðòÒ̲Ǥ gpu.0.gpu‚90.0 gpu.0.memory‚20, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚48.0 gpu.0.powerWatts‚114.426' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚38.141999999999996 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0én1`Ë‚(:Õ ÝðòÒ†ÝФ network.recv‚326961% disk./.usageGB‚1171.1250839233398& disk.nvme0n1.out‚18246.38623046875$ memory_percent‚99.60689544677734* proc.memory.percent‚17.258071899414062) disk./.usagePercent‚76.28236990218791# disk.dm-0.out‚28540.37060546875 disk.rbd0.out‚0( disk.nvme0c0n1.out‚18246.38623046875 disk.nvme1c1n1.in‚ 129.765625$ disk.nvme1c1n1.out‚ 11272.7265625" disk.nvme1n1.out‚ 11272.7265625 disk.md0.in‚0 disk.dm-0.in‚ 1982.2109375 disk.dm-1.out‚ 917.1640625 disk.rbd0.in‚0" disk.nvme0c0n1.in‚ 1852.4453125 disk.nvme1n1.in‚ 129.765625 disk.md0.out‚28.75$ proc.memory.availableMB‚64.40625 proc.memory.rssMB‚ 2827.5625 disk.dm-1.in‚0 disk.nvme0n1.in‚ 1852.4453125 cpu‚25.779784821356145 proc.cpu.threads‚17 network.sent‚ 2.250661e+06lž£;2Ì- _wandbruntime‚1087  _runtime‚1087[‰|aÇÍÁ trainer/global_step‚1599 % _timestamp‚1.7844614132434676e+09 _runtime‚1087.633827928 _step‚127 & lr-AdamW/pg1‚0.0001379608475496789 & lr-AdamW/pg2‚0.0001379608475496789ÞšSËÎÅ _step‚127 & lr-AdamW/pg1‚0.0001379608475496789 & lr-AdamW/pg2‚0.0001379608475496789 trainer/global_step‚1599 % _timestamp‚1.7844614132434676e+09 _runtime‚1087.633827928é:q2Ï- _wandbruntime‚1087  _runtime‚1087u{Ñîå$ÐÔO GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1345640532992 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1302627714924 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚28591687 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚12575771 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚180 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚3590730 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚93621231692288 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚123248097171968 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚124554350732800 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚121803662356688 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚12575773 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1306147861504 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚28601832 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚16026059 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚123106307110972 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1306147861504 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚121803679396048 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚184 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1302540928498 % _timestamp‚1.7844614132441306e+09 _step‚128 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚123248097171968 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 221456384 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 127346688 trainer/global_step‚1599 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚124554245033472 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1306236521472 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚16026059 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1306236521472 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TEDeviceStatsMbS¹õonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372523008 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1345513186304 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚123248114211328 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚123106203285186 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚6825â4®ß'ÑÙO TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚16015916 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚121803679396048 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚184 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1302540928498 % _timestamp‚1.7844614132441306e+09 _step‚128 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚123248097171968 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 221456384 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 127346688 trainer/global_step‚1599 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚124554245033472 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1306236521472 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚16026059 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1306236521472 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372523008 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1345513186304 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚123248114211328 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚123106203285186 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1345640532992 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1302627714924 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚28591687 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚12575771 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚180 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚3590730 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚93621231692288 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚123248097171968 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚124554350732800 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚121803662356688 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚12575773 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1306147861504 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚28601832 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚16026059 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚123106307110972 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1306147861504 IDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚28609136 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚16035893 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚8420722 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚12583402 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚28619295 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 $ _timestamp‚1.784461413649939e+09 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚16035893 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚12583402 trainer/global_step‚1599 _runtime‚1088.038472895 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚93678175090688 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3592825 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚124630180415488 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1306933577216 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚12583400 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚121877952882080 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1303411209636 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372523008 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1307022244352 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚121877969921440 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚28619295 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚3592996 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚123181277305178 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚12013543 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚92331856233984 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚124630286121984 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚92331950343680 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚123323263877632 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚123181381131076 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚123323246838272 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1307022244352 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚123323263877632 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚16025736 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚171 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚93678402831360 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚175 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1346318856704 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚12583400 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚123323246838272 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 227740672 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1303324423098kœÆŸè&ÔâM PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚171 [HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚93678402831360 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚175 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1346318856704 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚12583400 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚123323246838272 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 227740672 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1303324423098 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚28609136 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚16035893 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚8420722 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚12583402 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚28619295 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 $ _timestamp‚1.784461413649939e+09 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚16035893 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚12583402 trainer/global_step‚1599 _runtime‚1088.038472895 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 WDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚93678175090688 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3592825 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚124630180415488 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1306933577216 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚12583400 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚121877952882080 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1303411209636 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372523008 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1307022244352 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚121877969921440 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚28619295 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚3592996 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚123181277305178 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚12013543 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚92331856233984 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚124630286121984 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚92331950343680 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚123323263877632 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚123181381131076 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚123323246838272 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1307022244352 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚123323263877632 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚16025736 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 PpœÙ‚(:‘ ìðòÒÀšî¤ gpu.0.gpu‚34.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚48.0 gpu.0.powerWatts‚162.53' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚54.17666666666666 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0à–2Ú- _wandbruntime‚1102  _runtime‚1102}9¸1>Û«| :"Purity/1FRt*SRqqt*FBt/event_purity‚0.08017219603061676 ("Purity/1FRt1SRqqt2FBt/event_purity‚0 ,&Purity/2FRt2SRqqt2FBt/event_proportion‚0 7 Purity/*FRt*SRqqt2FBt/FRt_purity‚0.7378306984901428 =&Purity/*FRt0SRqqt2FBt/event_proportion‚0.0103004090487957 @&Purity/*FRt1SRqqt2FBt/event_proportion‚0.0006975446594879031 7 Purity/0FRt1SRqqt1FBt/FBt_purity‚0.9803376197814941 9"Purity/2FRt0SRqqt1FBt/event_purity‚0.2083333432674408 ("Purity/2FRt0SRqqt2FBt/event_purity‚0 jet/accuracy_3_of_5‚0 7 Purity/*FRt1SRqqt*FBt/FRt_purity‚0.3639838397502899 9"Purity/*FRt1SRqqt0FBt/SRqqt_purity‚0.1165054515004158 8&Purity/*FRt2SRqqt0FBt/event_proportion‚ 0.05322265625 7 Purity/1FRt*SRqqt2FBt/FBt_purity‚0.9575113654136658 8 Purity/1FRt0SRqqt*FBt/FRt_purity‚0.11564461141824722 9"Purity/1FRt1SRqqt2FBt/SRqqt_purity‚0.3888888955116272 +jet/accuracy_3_of_4‚0.04321863129734993 epoch‚3 =&Purity/0FRt1SRqqt0FBt/event_proportion‚0.2083565890789032 7 Purity/1FRt*SRqqt2FBt/FRt_purity‚0.7875283360481262 9"Purity/1FRt2SRqqt1FBt/SRqqt_purity‚0.3499999940395355 A&Purity/2FRt0SRqqt2FBt/event_proportion‚0.00018601190822664648 ("Purity/2FRt*SRqqt2FBt/event_purity‚0 7 Purity/*FRt*SRqqt2FBt/FBt_purity‚0.9377917051315308 >&Purity/1FRt0SRqqt1FBt/event_proportion‚0.00969587080180645 & Purity/1FRt2SRqqt1FBt/FBt_purity‚1 @&Purity/2FRt0SRqqt1FBt/event_proportion‚0.0007672990905120969 particle/accuracy_1_of_1‚1 9"Purity/*FRt*SRqqt*FBt/SRqqt_purity‚0.1713324785232544 :"Purity/*FRt*SRqqt0FBt/event_purity‚0.08155428618192673 :"Purity/*FRt1SRqqt*FBt/SRqqt_purity‚0.13798819482326508 ("Purity/1FRt2SRqqt*FBt/event_purity‚0 8&Purity/1FRt2SRqqt0FBt/event_proportion‚ 0.02294921875 :"Purity/2FRt2SRqqt*FBt/SRqqt_purity‚0.31475886702537537 ("Purity/2FRt*SRqqt2FBt/SRqqt_purity‚0 7 Purity/*FRt0SRqqt2FBt/FRt_purity‚0.7374905347824097 =&Purity/1FRt1SRqqt*FBt/event_proportion‚0.1534598171710968 & Purity/2FRt*SRqqt1FBt/FBt_purity‚1 & Purity/2FRt1SRqqt2FBt/FRt_purity‚0 ;"Purity/*FRt2SRqqt*FBt/event_purity‚0.024101609364151955 =&Purity/0FRt*SRqqt1FBt/event_proportion‚0.0436662957072258 jet/accuracy_5_of_5‚0 :"Purity/*FRt2SRqqt0FBt/SRqqt_purity‚0.29174137115478516 + Purity/2FRt*SRqqt2FBt/FBt_purity‚0.9375 @&Purity/1FRt*SRqqt2FBt/event_proportion‚0.0027204242069274187 :"Purity/2FRt1SRqqt*FBt/SRqqt_purity‚0.17597396671772003 7 Purity/2FRt2SRqqt0FBt/FRt_purity‚0.2180921733379364 (particle/accuracy‚0.8066754937171936 7 Purity/*FRt*SRqqt1FBt/FBt_purity‚0.9878616333007812 :"Purity/1FRt*SRqqt1FBt/event_purity‚0.19295480847358704 :"Purity/1FRt1SRqqt1FBt/event_purity‚0.06028657779097557 @&Purity/2FRt1SRqqt2FBt/event_proportion‚2.325148852833081e-05 9"Purity/*FRt0SRqqt0FBt/event_purity‚0.0981694757938385 9"Purity/1FRt2SRqqt0FBt/SRqqt_purity‚0.3223596215248108 9"Purity/2FRt*SRqqt1FBt/event_purity‚0.0950000062584877 +jet/accuracy_2_of_2‚0.06833537667989731 particle/accuracy_2_of_2‚1 7 Purity/0FRt1SRqqt2FBt/FBt_purity‚0.6538461446762085 9"Purity/*FRt1SRqqt1FBt/event_purity‚0.2149934470653534 :"Purity/0FRt1SRqqt0FBt/SRqqt_purity‚0.11724667251110077 7 Purity/1FRt*SRqqt1FBt/FBt_purity‚0.9932321906089783 particle/accuracy_2_of_3‚1 particle/accuracy_1_of_5‚1 9"Purity/*FRt0SRqqt*FBt/event_purity‚0.1701316237449646 7 Purity/*FRt0SRqqt1FBt/FBt_purity‚0.9890291094779968 :"Purity/0FRt1SRqqt1FBt/event_purity‚0.31767717003822327 , Purity/0FRt2SRqqt*FBt/FBt_purity‚0.96875 & Purity/2FRt1SRqqt1FBt/FBt_purity‚1 , Purity/2FRt*SRqqt2FBt/FRt_purity‚0.28125 particle/accuracy_5_of_5‚0 9"Purity/*FRt*SRqqt1FBt/event_purity‚0.6153239607810974 =&Purity/1FRt0SRqqt*FBt/event_proportion‚0.4530552327632904 =&Purity/0FRt0SRqqt1FBt/event_proportion‚0.0313895083963871 >&Purity/2FRt0SRqqt0FBt/event_proportion‚0.01990327425301075 'particle/f_score‚0.5376583933830261 <&Purity/*FRt*SRqqt0FBt/event_proportion‚0.881045401096344 7 Purity/*FRt1SRqqt2FBt/FBt_purity‚0.6342592239379883 8 Purity/1FRt*SRqqt*FBt/FRt_purity‚0.18270127475261688 7 Purity/1FRt0SRqqt2FBt/FBt_purity‚0.9851190447807312 @&Purity/2FRt*SRqqt1FBt/event_proportion‚0.0016043527284637094 ("Purity/2FRt1SRqqt*FBt/event_purity‚0 8 Purity/*FRt0SRqqt*FBt/FRt_purity‚0.11552548408508301 :"Purity/*FRt2SRqqt0FBt/event_purity‚0.02462811768054962 7 Purity/2FRt1SRqqt*FBt/FRt_purity‚0.3292335271835327 :"Purity/*FRt2SRqqt*FBt/SRqqt_purity‚0.29067569971084595 =&Purity/1FRt1SRqqt0FBt/event_proportion‚0.1467633992433548 >&Purity/1FRt2SRqqt*FBt/event_proportion‚0.02322823740541935 jet/accuracy_4_of_5‚0 trainer/global_step‚1599 ("Purity/1FRt2SRqqt1FBt/event_purity‚0 @&Purity/2FRt1SRqqt1FBt/event_proportion‚0.0008370535797439516 & Purity/2FRt0SRqqt*FBt/FBt_purity‚1 6 Purity/*FRt1SRqqt2FBt/FRt_purity‚0.699999988079071 :"Purity/0FRt1SRqqt0FBt/event_purity‚0.11724667251110077 7 Purity/1FRt0SRqqt2FBt/FRt_purity‚0.7781179547309875 7 Purity/1FRt1SRqqt0FBt/FRt_purity‚0.3610991835594177 7 Purity/1FRt1SRqqt1FBt/FRt_purity‚0.5439409613609314 +jet/accuracy_1_of_2‚0.46207568049430847 _runtime‚1102.222485007 <&Purity/*FRt2SRqqt*FBt/event_proportion‚0.054222472012043 8 Purity/*FRt*SRqqt*FBt/FRt_purity‚0.18453934788703918 8 Purity/*FRt2SRqqt0FBt/FRt_purity‚0.24532783031463623 8 Purity/*FRt2SRqqt1FBt/FRt_purity‚0.10000000149011612 =&Purity/1FRt*SRqqt0FBt/event_proportion‚0.6105840802192688 ("Purity/1FRt2SRqqt0FBt/event_purity‚0 ,&Purity/1FRt2SRqqt2FBt/event_proportion‚0 +jet/accuracy_3_of_3‚0.03093011863529682 *loss/val_total_loss‚15.950676502531115 9"Purity/*FRt0SRqqt1FBt/event_purity‚0.8115033507347107 :"Purity/*FRt1SRqqt0FBt/event_purity‚0.06916407495737076 9"Purity/1FRt0SRqqt0FBt/event_purity‚0.1023736447095871 7 Purity/1FRt1SRqqt*FBt/FBt_purity‚0.9733588695526123 particle/accuracy_4_of_5‚1 7 Purity/*FRt2SRqqt*FBt/FBt_purity‚0.9814814329147339 9"Purity/0FRt0SRqqt2FBt/event_purity‚0.8926917314529419 =&Purity/0FRt1SRqqt*FBt/event_proportion‚0.2203543484210968 =&Purity/1FRt0SRqqt0FBt/event_proportion‚0.4408714771270752 7 Purity/1FRt2SRqqt*FBt/FRt_purity‚0.2582099437713623 7 Purity/0FRt0SRqqt*FBt/FBt_purity‚0.9750966429710388 :"Purity/0FRt1SRqqt*FBt/SRqqt_purity‚0.13923069834709167 :"Purity/1FRt*SRqqt*FBt/SRqqt_purity‚0.17658349871635437 9"Purity/1FRt*SRqqt1FBt/SRqqt_purity‚0.5498005747795105 9"Purity/1FRt1SRqqt1FBt/SRqqt_purity‚0.5630343556404114 +jet/accuracy_1_of_1‚0.12236304581165314 9"Purity/0FRt*SRqqt0FBt/SRqqt_purity‚0.1435893326997757 8 Purity/2FRt*SRqqt*FBt/FRt_purity‚0.19957247376441956 =&Purity/2FRt*SRqqt0FBt/event_proportion‚0.0382486991584301 8 Purity/2FRt0SRqqt0FBt/FRt_purity‚0.10142451524734497 7 Purity/2FRt0SRqqt1FBt/FRt_purity‚0.4244791567325592 =&Purity/*FRt1SRqqt*FBt/event_proportion‚0.3866024911403656 9"Purity/*FRt1SRqqt1FBt/SRqqt_purity‚0.5458424687385559 ?&Purity/0FRt1SRqqt1FBt/event_proportion‚0.011555989272892475 @&Purity/1FRt1SRqqt2FBt/event_proportion‚0.0002325148816453293 :"Purity/*FRt*SRqqt*FBt/event_purity‚0.12372181564569473 :"Purity/*FRt*SRqqt0FBt/SRqqt_purity‚0.15579310059547424 :"Purity/*FRt1SRqqt*FBt/event_purity‚0.07618643343448639 9"Purity/0FRt0SRqqt*FBt/event_purity‚0.9701952338218689 :"Purity/1FRt2SRqqt*FBt/SRqqt_purity‚0.32232728600502014 +"Purity/2FRt*SRqqt1FBt/SRqqt_purity‚0.75 ("Purity/2FRt2SRqqt*FBt/event_purity‚0 9"Purity/*FRt*SRqqt1FBt/SRqqt_purity‚0.5152085423469543 7 Purity/0FRt*SRqqt*FBt/FBt_purity‚0.9718157649040222 +particle/sensitivity‚0.5234674215316772 7 Purity/*FRt1SRqqt1FBt/FBt_purity‚0.9852949976921082 9"Purity/0FRt1SRqqt1FBt/SRqqt_purity‚0.5268680453300476 9"Purity/1FRt0SRqqt1FBt/event_purity‚0.2850925028324127 7 Purity/1FRt0SRqqt1FBt/FBt_purity‚0.9918831586837769 *jet/accuracy_1_of_4‚0.8117995262145996 particle/accuracy_1_of_4‚1 ("Purity/0FRt2SRqqt1FBt/event_purity‚0 +jet/accuracy_2_of_4‚0.31775712966918945 /particle/accuracy_3_of_4‚0.9968253374099731 +particle/specificity‚0.8841313123703003 & Purity/2FRt0SRqqt2FBt/FBt_purity‚1 =&Purity/*FRt1SRqqt0FBt/event_proportion‚0.3670479953289032 7 Purity/0FRt0SRqqt1FBt/FBt_purity‚0.9877130389213562 ;"Purity/2FRt0SRqqt0FBt/event_purity‚0.004265128634870052 @&Purity/*FRt2SRqqt1FBt/event_proportion‚0.0009998140158131719 7 Purity/*FRt2SRqqt1FBt/FBt_purity‚0.9814814329147339 ,&Purity/*FRt2SRqqt2FBt/event_proportion‚0 :"Purity/0FRt*SRqqt*FBt/event_purity‚0.23678913712501526 :"Purity/1FRt0SRqqt*FBt/event_purity‚0.10851605981588364 jet/accuracy_2_of_5‚0 7 Purity/1FRt1SRqqt1FBt/FBt_purity‚0.9942227602005005 7 Purity/2FRt2SRqqt*FBt/FRt_purity‚0.2180921733379364 7 Purity/*FRt1SRqqt*FBt/FBt_purity‚0.9636979103088379 :"Purity/1FRt*SRqqt0FBt/SRqqt_purity‚0.16280077397823334 7 Purity/1FRt0SRqqt*FBt/FBt_purity‚0.9895561933517456 @&Purity/2FRt*SRqqt2FBt/event_proportion‚0.0002092633949359879 :"Purity/2FRt2SRqqt0FBt/SRqqt_purity‚0.31475886702537537 =&Purity/*FRt*SRqqt*FBt/event_proportion‚0.9537528157234192 7 Purity/0FRt*SRqqt1FBt/FBt_purity‚0.9854196906089783 =&Purity/2FRt0SRqqt*FBt/event_proportion‚0.0208565853536129 ("Purity/2FRt1SRqqt2FBt/SRqqt_purity‚0 7 Purity/2FRt0SRqqt2FBt/FRt_purity‚0.3214285671710968 ("Purity/2FRt1SRqqt2FBt/event_purity‚0 @&Purity/0FRt2SRqqt1FBt/event_proportion‚0.0007207961170934141 particle/accuracy_1_of_2‚1 /particle/accuracy_4_of_4‚0.9255225658416748 =&Purity/0FRt*SRqqt0FBt/event_proportion‚0.2322126179933548 :"Purity/0FRt*SRqqt1FBt/SRqqt_purity‚0.48965132236480713 :"Purity/1FRt1SRqqt*FBt/event_purity‚0.00859908014535904 :"Purity/1FRt1SRqqt0FBt/event_purity‚0.00625558290630579 7 Purity/*FRt1SRqqt0FBt/FRt_purity‚0.3561079204082489 6 Purity/*FRt1SRqqt1FBt/FRt_purity‚0.522887110710144 ("Purity/*FRt2SRqqt1FBt/event_purity‚0 ;"Purity/0FRt2SRqqt0FBt/event_purity‚0.055943019688129425 7 Purity/1FRt0SRqqt0FBt/FRt_purity‚0.1023736447095871 7 Purity/1FRt0SRqqt1FBt/FRt_purity‚0.5415266752243042 7 Purity/1FRt1SRqqt2FBt/FRt_purity‚0.7777777910232544 <&Purity/2FRt*SRqqt*FBt/event_proportion‚0.040062315762043 7 Purity/0FRt*SRqqt2FBt/FBt_purity‚0.9319707751274109 ("Purity/0FRt0SRqqt0FBt/event_purity‚1 >&Purity/0FRt2SRqqt*FBt/event_proportion‚0.02457682229578495 9"Purity/0FRt2SRqqt*FBt/SRqqt_purity‚0.2579576075077057 =&Purity/1FRt*SRqqt*FBt/event_proportion‚0.6297432780265808 >&Purity/1FRt*SRqqt1FBt/event_proportion‚0.01643880270421505 ;"Purity/2FRt*SRqqt*FBt/event_purity‚0.006241144146770239 ;"Purity/2FRt*SRqqt0FBt/event_purity‚0.002370529342442751 +"Purity/2FRt1SRqqt1FBt/SRqqt_purity‚0.75 ?&Purity/2FRt2SRqqt*FBt/event_proportion‚0.006417410913854837 ?&Purity/2FRt2SRqqt0FBt/event_proportion‚0.006417410913854837 particle/accuracy_2_of_4‚1 7 Purity/1FRt1SRqqt2FBt/FBt_purity‚0.6388888955116272 =&Purity/2FRt1SRqqt*FBt/event_proportion‚0.0127883180975914 ?&Purity/2FRt1SRqqt0FBt/event_proportion‚0.011928013525903225 +jet/accuracy_2_of_3‚0.26754212379455566 jet/accuracy_1_of_5‚1 & Purity/2FRt0SRqqt1FBt/FBt_purity‚1 7 Purity/0FRt0SRqqt2FBt/FBt_purity‚0.9463458061218262 ("Purity/2FRt1SRqqt0FBt/event_purity‚0 7 Purity/*FRt*SRqqt*FBt/FBt_purity‚0.9752777814865112 :"Purity/0FRt2SRqqt*FBt/event_purity‚0.05401565879583359 :"Purity/1FRt*SRqqt2FBt/event_purity‚0.46774378418922424 8 Purity/1FRt2SRqqt0FBt/FRt_purity‚0.26089149713516235 8 Purity/1FRt2SRqqt1FBt/FRt_purity‚0.10000000149011612 9"Purity/2FRt*SRqqt*FBt/SRqqt_purity‚0.2441549003124237 :"Purity/2FRt*SRqqt0FBt/SRqqt_purity‚0.22837689518928528 9"Purity/0FRt*SRqqt2FBt/SRqqt_purity‚0.4615384638309479 >&Purity/0FRt2SRqqt0FBt/event_proportion‚0.02385602705180645 :"Purity/0FRt2SRqqt0FBt/SRqqt_purity‚0.25990593433380127 ."Purity/0FRt2SRqqt1FBt/SRqqt_purity‚0.21875 9"Purity/0FRt*SRqqt2FBt/event_purity‚0.8403047919273376 =&Purity/0FRt0SRqqt*FBt/event_proportion‚0.0390159972012043 @&Purity/0FRt1SRqqt2FBt/event_proportion‚0.0004417782765813172 ,&Purity/0FRt2SRqqt2FBt/event_proportion‚0 9"Purity/1FRt*SRqqt2FBt/SRqqt_purity‚0.3888888955116272 :"Purity/1FRt1SRqqt*FBt/SRqqt_purity‚0.13301950693130493 :"Purity/1FRt1SRqqt0FBt/SRqqt_purity‚0.11355645209550858 6 Purity/2FRt*SRqqt0FBt/FRt_purity‚0.190482497215271 =&Purity/*FRt0SRqqt0FBt/event_proportion‚0.4607747495174408 =&Purity/*FRt1SRqqt1FBt/event_proportion‚0.0188569575548172 5 Purity/2FRt*SRqqt1FBt/FRt_purity‚0.42208331823349 8 Purity/2FRt0SRqqt*FBt/FRt_purity‚0.11458112299442291 ,&Purity/2FRt2SRqqt1FBt/event_proportion‚0 =&Purity/*FRt*SRqqt1FBt/event_proportion‚0.0617094486951828 9"Purity/*FRt*SRqqt2FBt/event_purity‚0.7327017784118652 ("Purity/*FRt1SRqqt2FBt/event_purity‚0 9"Purity/2FRt1SRqqt0FBt/SRqqt_purity‚0.1361302137374878 9"Purity/0FRt1SRqqt2FBt/SRqqt_purity‚0.4615384638309479 7 Purity/*FRt0SRqqt*FBt/FBt_purity‚0.9789225459098816 , Purity/0FRt2SRqqt1FBt/FBt_purity‚0.96875 7 Purity/2FRt1SRqqt*FBt/FBt_purity‚0.9736841917037964 particle/accuracy_2_of_5‚1 9"Purity/*FRt1SRqqt2FBt/SRqqt_purity‚0.4444444477558136 9"Purity/*FRt*SRqqt2FBt/SRqqt_purity‚0.4444444477558136 =&Purity/0FRt0SRqqt0FBt/event_proportion‚0.0462472103536129 7 Purity/1FRt*SRqqt*FBt/FBt_purity‚0.9835972785949707 ("Purity/0FRt1SRqqt2FBt/event_purity‚0 _step‚131 =&Purity/*FRt0SRqqt1FBt/event_proportion‚0.0418526791036129 7validation_average_jet_accuracy‚0.16711869835853577 9"Purity/0FRt*SRqqt1FBt/event_purity‚0.7923840284347534 7 Purity/1FRt*SRqqt0FBt/FRt_purity‚0.1703893393278122 7 Purity/1FRt*SRqqt1FBt/FRt_purity‚0.5369454026222229 :"Purity/0FRt*SRqqt*FBt/SRqqt_purity‚0.16075411438941956 7 Purity/0FRt1SRqqt*FBt/FBt_purity‚0.9604708552360535 A&Purity/1FRt2SRqqt1FBt/event_proportion‚0.00027901786961592734 jet/accuracy_4_of_4‚0 ?&Purity/*FRt*SRqqt2FBt/event_proportion‚0.010997953824698925 ?&Purity/0FRt*SRqqt2FBt/event_proportion‚0.008068266324698925 9"Purity/0FRt0SRqqt1FBt/event_purity‚0.9877130389213562 ("Purity/2FRt1SRqqt1FBt/event_purity‚0 % _timestamp‚1.7844614278328469e+09 8 Purity/*FRt0SRqqt0FBt/FRt_purity‚0.10223440825939178 6 Purity/*FRt0SRqqt1FBt/FRt_purity‚0.524236798286438 9"Purity/*FRt2SRqqt1FBt/SRqqt_purity‚0.2638888955116272 7 Purity/2FRt1SRqqt0FBt/FRt_purity‚0.3258647620677948 8 Purity/2FRt1SRqqt1FBt/FRt_purity‚0.41203704476356506 *jet/accuracy_1_of_3‚0.7273964881896973 =&Purity/0FRt*SRqqt*FBt/event_proportion‚0.2839471697807312 ?&Purity/0FRt0SRqqt2FBt/event_proportion‚0.007626487873494625 @&Purity/1FRt0SRqqt2FBt/event_proportion‚0.0024879092816263437 6 Purity/2FRt*SRqqt*FBt/FBt_purity‚0.987500011920929 @&Purity/1FRt1SRqqt1FBt/event_proportion‚0.0064639137126505375 7 Purity/*FRt*SRqqt0FBt/FRt_purity‚0.1725425124168396 7 Purity/*FRt*SRqqt1FBt/FRt_purity‚0.5170144438743591 =&Purity/*FRt0SRqqt*FBt/event_proportion‚0.5129278302192688 8 Purity/*FRt2SRqqt*FBt/FRt_purity‚0.24406524002552032 & Purity/1FRt2SRqqt*FBt/FBt_purity‚1 /particle/accuracy_3_of_3‚0.9990025758743286 particle/accuracy_3_of_5‚1 :"Purity/1FRt*SRqqt0FBt/event_purity‚0.07543495297431946 9"Purity/1FRt0SRqqt2FBt/event_purity‚0.5026644468307495 8 Purity/1FRt1SRqqt*FBt/FRt_purity‚0.36963316798210144 ;"Purity/2FRt0SRqqt*FBt/event_purity‚0.011851946823298931 ("Purity/2FRt2SRqqt0FBt/event_purity‚0 particle/accuracy_1_of_3‚1 7 Purity/*FRt0SRqqt2FBt/FBt_purity‚0.9564594030380249 :"Purity/0FRt*SRqqt0FBt/event_purity‚0.11076541244983673 ( Purity/2FRt1SRqqt2FBt/FBt_purity‚0.5 9"Purity/*FRt0SRqqt2FBt/event_purity‚0.7837355732917786 9"Purity/0FRt1SRqqt*FBt/event_purity‚0.1275300681591034;Ø¢'=Ü°| ("Purity/0FRt1SRqqt2FBt/event_purity‚0 _step‚131 =&Purity/*FRt0SRqqt1FBt/event_proportion‚0.0418526791036129 7validation_average_jet_accuracy‚0.16711869835853577 9"Purity/0FRt*SRqqt1FBt/event_purity‚0.7923840284347534 7 Purity/1FRt*SRqqt0FBt/FRt_purity‚0.1703893393278122 7 Purity/1FRt*SRqqt1FBt/FRt_purity‚0.5369454026222229 :"Purity/0FRt*SRqqt*FBt/SRqqt_purity‚0.16075411438941956 7 Purity/0FRt1SRqqt*FBt/FBt_purity‚0.9604708552360535 A&Purity/1FRt2SRqqt1FBt/event_proportion‚0.00027901786961592734 jet/accuracy_4_of_4‚0 ?&Purity/*FRt*SRqqt2FBt/event_proportion‚0.010997953824698925 ?&Purity/0FRt*SRqqt2FBt/event_proportion‚0.008068266324698925 9"Purity/0FRt0SRqqt1FBt/event_purity‚0.9877130389213562 ("Purity/2FRt1SRqqt1FBt/event_purity‚0 % _timestamp‚1.7844614278328469e+09 8 Purity/*FRt0SRqqt0FBt/FRt_purity‚0.10223440825939178 6 Purity/*FRt0SRqqt1FBt/FRt_purity‚0.524236798286438 9"Purity/*FRt2SRqqt1FBt/SRqqt_purity‚0.2638888955116272 7 Purity/2FRt1SRqqt0FBt/FRt_purity‚0.3258647620677948 8 Purity/2FRt1SRqqt1FBt/FRt_purity‚0.41203704476356506 *jet/accuracy_1_of_3‚0.7273964881896973 =&Purity/0FRt*SRqqt*FBt/event_proportion‚0.2839471697807312 ?&Purity/0FRt0SRqqt2FBt/event_proportion‚0.007626487873494625 @&Purity/1FRt0SRqqt2FBt/event_proportion‚0.0024879092816263437 6 Purity/2FRt*SRqqt*FBt/FBt_purity‚0.987500011920929 @&Purity/1FRt1SRqqt1FBt/event_proportion‚0.0064639137126505375 7 Purity/*FRt*SRqqt0FBt/FRt_purity‚0.1725425124168396 7 Purity/*FRt*SRqqt1FBt/FRt_purity‚0.5170144438743591 =&Purity/*FRt0SRqqt*FBt/event_proportion‚0.5129278302192688 8 Purity/*FRt2SRqqt*FBt/FRt_purity‚0.24406524002552032 & Purity/1FRt2SRqqt*FBt/FBt_purity‚1 /particle/accuracy_3_of_3‚0.9990025758743286 particle/accuracy_3_of_5‚1 :"Purity/1FRt*SRqqt0FBt/event_purity‚0.07543495297431946 9"Purity/1FRt0SRqqt2FBt/event_purity‚0.5026644468307495 8 Purity/1FRt1SRqqt*FBt/FRt_purity‚0.36963316798210144 ;"Purity/2FRt0SRqqt*FBt/event_purity‚0.011851946823298931 ("Purity/2FRt2SRqqt0FBt/event_purity‚0 particle/accuracy_1_of_3‚1 7 Purity/*FRt0SRqqt2FBt/FBt_purity‚0.9564594030380249 :"Purity/0FRt*SRqqt0FBt/event_purity‚0.11076541244983673 ( Purity/2FRt1SRqqt2FBt/FBt_purity‚0.5 9"Purity/*FRt0SRqqt2FBt/event_purity‚0.7837355732917786 9"Purity/0FRt1SRqqt*FBt/event_purity‚0.1275300681591034 :"Purity/1FRt*SRqqt*FBt/event_purity‚0.08017219603061676 ("Purity/1FRt1SRqqt2FBt/event_purity‚0 ,&Purity/2FRt2SRqqt2FBt/event_proportion‚0 7 Purity/*FRt*SRqqt2FBt/FRt_purity‚0.7378306984901428 =&Purity/*FRt0SRqqt2FBt/event_proportion‚0.0103004090487957 @&Purity/*FRt1SRqqt2FBt/event_proportion‚0.0006975446594879031 7 Purity/0FRt1SRqqt1FBt/FBt_purity‚0.9803376197814941 9"Purity/2FRt0SRqqt1FBt/event_purity‚0.2083333432674408 ("Purity/2FRt0SRqqt2FBt/event_purity‚0 jet/accuracy_3_of_5‚0 7 Purity/*FRt1SRqqt*FBt/FRt_purity‚0.3639838397502899 9"Purity/*FRt1SRqqt0FBt/SRqqt_purity‚0.1165054515004158 8&Purity/*FRt2SRqqt0FBt/event_proportion‚ 0.05322265625 7 Purity/1FRt*SRqqt2FBt/FBt_purity‚0.9575113654136658 8 Purity/1FRt0SRqqt*FBt/FRt_purity‚0.11564461141824722 9"Purity/1FRt1SRqqt2FBt/SRqqt_purity‚0.3888888955116272 +jet/accuracy_3_of_4‚0.04321863129734993 epoch‚3 =&Purity/0FRt1SRqqt0FBt/event_proportion‚0.2083565890789032 7 Purity/1FRt*SRqqt2FBt/FRt_purity‚0.7875283360481262 9"Purity/1FRt2SRqqt1FBt/SRqqt_purity‚0.3499999940395355 A&Purity/2FRt0SRqqt2FBt/event_proportion‚0.00018601190822664648 ("Purity/2FRt*SRqqt2FBt/event_purity‚0 7 Purity/*FRt*SRqqt2FBt/FBt_purity‚0.9377917051315308 >&Purity/1FRt0SRqqt1FBt/event_proportion‚0.00969587080180645 & Purity/1FRt2SRqqt1FBt/FBt_purity‚1 @&Purity/2FRt0SRqqt1FBt/event_proportion‚0.0007672990905120969 particle/accuracy_1_of_1‚1 9"Purity/*FRt*SRqqt*FBt/SRqqt_purity‚0.1713324785232544 :"Purity/*FRt*SRqqt0FBt/event_purity‚0.08155428618192673 :"Purity/*FRt1SRqqt*FBt/SRqqt_purity‚0.13798819482326508 ("Purity/1FRt2SRqqt*FBt/event_purity‚0 8&Purity/1FRt2SRqqt0FBt/event_proportion‚ 0.02294921875 :"Purity/2FRt2SRqqt*FBt/SRqqt_purity‚0.31475886702537537 ("Purity/2FRt*SRqqt2FBt/SRqqt_purity‚0 7 Purity/*FRt0SRqqt2FBt/FRt_purity‚0.7374905347824097 =&Purity/1FRt1SRqqt*FBt/event_proportion‚0.1534598171710968 & Purity/2FRt*SRqqt1FBt/FBt_purity‚1 & Purity/2FRt1SRqqt2FBt/FRt_purity‚0 ;"Purity/*FRt2SRqqt*FBt/event_purity‚0.024101609364151955 =&Purity/0FRt*SRqqt1FBt/event_proportion‚0.0436662957072258 jet/accuracy_5_of_5‚0 :"Purity/*FRt2SRqqt0FBt/SRqqt_purity‚0.29174137115478516 + Purity/2FRt*SRqqt2FBt/FBt_purity‚0.9375 @&Purity/1FRt*SRqqt2FBt/event_proportion‚0.0027204242069274187 :"Purity/2FRt1SRqqt*FBt/SRqqt_purity‚0.17597396671772003 7 Purity/2FRt2SRqqt0FBt/FRt_purity‚0.2180921733379364 (particle/accuracy‚0.8066754937171936 7 Purity/*FRt*SRqqt1FBt/FBt_purity‚0.9878616333007812 :"Purity/1FRt*SRqqt1FBt/event_purity‚0.19295480847358704 :"Purity/1FRt1SRqqt1FBt/event_purity‚0.06028657779097557 @&Purity/2FRt1SRqqt2FBt/event_proportion‚2.325148852833081e-05 9"Purity/*FRt0SRqqt0FBt/event_purity‚0.0981694757938385 9"Purity/1FRt2SRqqt0FBt/SRqqt_purity‚0.3223596215248108 9"Purity/2FRt*SRqqt1FBt/event_purity‚0.0950000062584877 +jet/accuracy_2_of_2‚0.06833537667989731 particle/accuracy_2_of_2‚1 7 Purity/0FRt1SRqqt2FBt/FBt_purity‚0.6538461446762085 9"Purity/*FRt1SRqqt1FBt/event_purity‚0.2149934470653534 :"Purity/0FRt1SRqqt0FBt/SRqqt_purity‚0.11724667251110077 7 Purity/1FRt*SRqqt1FBt/FBt_purity‚0.9932321906089783 particle/accuracy_2_of_3‚1 particle/accuracy_1_of_5‚1 9"Purity/*FRt0SRqqt*FBt/event_purity‚0.1701316237449646 7 Purity/*FRt0SRqqt1FBt/FBt_purity‚0.9890291094779968 :"Purity/0FRt1SRqqt1FBt/event_purity‚0.31767717003822327 , Purity/0FRt2SRqqt*FBt/FBt_purity‚0.96875 & Purity/2FRt1SRqqt1FBt/FBt_purity‚1 , Purity/2FRt*SRqqt2FBt/FRt_purity‚0.28125 particle/accuracy_5_of_5‚0 9"Purity/*FRt*SRqqt1FBt/event_purity‚0.6153239607810974 =&Purity/1FRt0SRqqt*FBt/event_proportion‚0.4530552327632904 =&Purity/0FRt0SRqqt1FBt/event_proportion‚0.0313895083963871 >&Purity/2FRt0SRqqt0FBt/event_proportion‚0.01990327425301075 'particle/f_score‚0.5376583933830261 <&Purity/*FRt*SRqqt0FBt/event_proportion‚0.881045401096344 7 Purity/*FRt1SRqqt2FBt/FBt_purity‚0.6342592239379883 8 Purity/1FRt*SRqqt*FBt/FRt_purity‚0.18270127475261688 7 Purity/1FRt0SRqqt2FBt/FBt_purity‚0.9851190447807312 @&Purity/2FRt*SRqqt1FBt/event_proportion‚0.0016043527284637094 ("Purity/2FRt1SRqqt*FBt/event_purity‚0 8 Purity/*FRt0SRqqt*FBt/FRt_purity‚0.11552548408508301 :"Purity/*FRt2SRqqt0FBt/event_purity‚0.02462811768054962 7 Purity/2FRt1SRqqt*FBt/FRt_purity‚0.3292335271835327 :"Purity/*FRt2SRqqt*FBt/SRqqt_purity‚0.29067569971084595 =&Purity/1FRt1SRqqt0FBt/event_proportion‚0.1467633992433548 >&Purity/1FRt2SRqqt*FBt/event_proportion‚0.02322823740541935 jet/accuracy_4_of_5‚0 trainer/global_step‚1599 ("Purity/1FRt2SRqqt1FBt/event_purity‚0 @&Purity/2FRt1SRqqt1FBt/event_proportion‚0.0008370535797439516 & Purity/2FRt0SRqqt*FBt/FBt_purity‚1 6 Purity/*FRt1SRqqt2FBt/FRt_purity‚0.699999988079071 :"Purity/0FRt1SRqqt0FBt/event_purity‚0.11724667251110077 7 Purity/1FRt0SRqqt2FBt/FRt_purity‚0.7781179547309875 7 Purity/1FRt1SRqqt0FBt/FRt_purity‚0.3610991835594177 7 Purity/1FRt1SRqqt1FBt/FRt_purity‚0.5439409613609314 +jet/accuracy_1_of_2‚0.46207568049430847 _runtime‚1102.222485007 <&Purity/*FRt2SRqqt*FBt/event_proportion‚0.054222472012043 8 Purity/*FRt*SRqqt*FBt/FRt_purity‚0.18453934788703918 8 Purity/*FRt2SRqqt0FBt/FRt_purity‚0.24532783031463623 8 Purity/*FRt2SRqqt1FBt/FRt_purity‚0.10000000149011612 =&Purity/1FRt*SRqqt0FBt/event_proportion‚0.6105840802192688 ("Purity/1FRt2SRqqt0FBt/event_purity‚0 ,&Purity/1FRt2SRqqt2FBt/event_proportion‚0 +jet/accuracy_3_of_3‚0.03093011863529682 *loss/val_total_loss‚15.950676502531115 9"Purity/*FRt0SRqqt1FBt/event_purity‚0.8115033507347107 :"Purity/*FRt1SRqqt0FBt/event_purity‚0.06916407495737076 9"Purity/1FRt0SRqqt0FBt/event_purity‚0.1023736447095871 7 Purity/1FRt1SRqqt*FBt/FBt_purity‚0.9733588695526123 particle/accuracy_4_of_5‚1 7 Purity/*FRt2SRqqt*FBt/FBt_purity‚0.9814814329147339 9"Purity/0FRt0SRqqt2FBt/event_purity‚0.8926917314529419 =&Purity/0FRt1SRqqt*FBt/event_proportion‚0.2203543484210968 =&Purity/1FRt0SRqqt0FBt/event_proportion‚0.4408714771270752 7 Purity/1FRt2SRqqt*FBt/FRt_purity‚0.2582099437713623 7 Purity/0FRt0SRqqt*FBt/FBt_purity‚0.9750966429710388 :"Purity/0FRt1SRqqt*FBt/SRqqt_purity‚0.13923069834709167 :"Purity/1FRt*SRqqt*FBt/SRqqt_purity‚0.17658349871635437 9"Purity/1FRt*SRqqt1FBt/SRqqt_purity‚0.5498005747795105 9"Purity/1FRt1SRqqt1FBt/SRqqt_purity‚0.5630343556404114 +jet/accuracy_1_of_1‚0.12236304581165314 9"Purity/0FRt*SRqqt0FBt/SRqqt_purity‚0.1435893326997757 8 Purity/2FRt*SRqqt*FBt/FRt_purity‚0.19957247376441956 =&Purity/2FRt*SRqqt0FBt/event_proportion‚0.0382486991584301 8 Purity/2FRt0SRqqt0FBt/FRt_purity‚0.10142451524734497 7 Purity/2FRt0SRqqt1FBt/FRt_purity‚0.4244791567325592 =&Purity/*FRt1SRqqt*FBt/event_proportion‚0.3866024911403656 9"Purity/*FRt1SRqqt1FBt/SRqqt_purity‚0.5458424687385559 ?&Purity/0FRt1SRqqt1FBt/event_proportion‚0.011555989272892475 @&Purity/1FRt1SRqqt2FBt/event_proportion‚0.0002325148816453293 :"Purity/*FRt*SRqqt*FBt/event_purity‚0.12372181564569473 :"Purity/*FRt*SRqqt0FBt/SRqqt_purity‚0.15579310059547424 :"Purity/*FRt1SRqqt*FBt/event_purity‚0.07618643343448639 9"Purity/0FRt0SRqqt*FBt/event_purity‚0.9701952338218689 :"Purity/1FRt2SRqqt*FBt/SRqqt_purity‚0.32232728600502014 +"Purity/2FRt*SRqqt1FBt/SRqqt_purity‚0.75 ("Purity/2FRt2SRqqt*FBt/event_purity‚0 9"Purity/*FRt*SRqqt1FBt/SRqqt_purity‚0.5152085423469543 7 Purity/0FRt*SRqqt*FBt/FBt_purity‚0.9718157649040222 +particle/sensitivity‚0.5234674215316772 7 Purity/*FRt1SRqqt1FBt/FBt_purity‚0.9852949976921082 9"Purity/0FRt1SRqqt1FBt/SRqqt_purity‚0.5268680453300476 9"Purity/1FRt0SRqqt1FBt/event_purity‚0.2850925028324127 7 Purity/1FRt0SRqqt1FBt/FBt_purity‚0.9918831586837769 *jet/accuracy_1_of_4‚0.8117995262145996 particle/accuracy_1_of_4‚1 ("Purity/0FRt2SRqqt1FBt/event_purity‚0 +jet/accuracy_2_of_4‚0.31775712966918945 /particle/accuracy_3_of_4‚0.9968253374099731 +particle/specificity‚0.8841313123703003 & Purity/2FRt0SRqqt2FBt/FBt_purity‚1 =&Purity/*FRt1SRqqt0FBt/event_proportion‚0.3670479953289032 7 Purity/0FRt0SRqqt1FBt/FBt_purity‚0.9877130389213562 ;"Purity/2FRt0SRqqt0FBt/event_purity‚0.004265128634870052 @&Purity/*FRt2SRqqt1FBt/event_proportion‚0.0009998140158131719 7 Purity/*FRt2SRqqt1FBt/FBt_purity‚0.9814814329147339 ,&Purity/*FRt2SRqqt2FBt/event_proportion‚0 :"Purity/0FRt*SRqqt*FBt/event_purity‚0.23678913712501526 :"Purity/1FRt0SRqqt*FBt/event_purity‚0.10851605981588364 jet/accuracy_2_of_5‚0 7 Purity/1FRt1SRqqt1FBt/FBt_purity‚0.9942227602005005 7 Purity/2FRt2SRqqt*FBt/FRt_purity‚0.2180921733379364 7 Purity/*FRt1SRqqt*FBt/FBt_purity‚0.9636979103088379 :"Purity/1FRt*SRqqt0FBt/SRqqt_purity‚0.16280077397823334 7 Purity/1FRt0SRqqt*FBt/FBt_purity‚0.9895561933517456 @&Purity/2FRt*SRqqt2FBt/event_proportion‚0.0002092633949359879 :"Purity/2FRt2SRqqt0FBt/SRqqt_purity‚0.31475886702537537 =&Purity/*FRt*SRqqt*FBt/event_proportion‚0.9537528157234192 7 Purity/0FRt*SRqqt1FBt/FBt_purity‚0.9854196906089783 =&Purity/2FRt0SRqqt*FBt/event_proportion‚0.0208565853536129 ("Purity/2FRt1SRqqt2FBt/SRqqt_purity‚0 7 Purity/2FRt0SRqqt2FBt/FRt_purity‚0.3214285671710968 ("Purity/2FRt1SRqqt2FBt/event_purity‚0 @&Purity/0FRt2SRqqt1FBt/event_proportion‚0.0007207961170934141 particle/accuracy_1_of_2‚1 /particle/accuracy_4_of_4‚0.9255225658416748 =&Purity/0FRt*SRqqt0FBt/event_proportion‚0.2322126179933548 :"Purity/0FRt*SRqqt1FBt/SRqqt_purity‚0.48965132236480713 :"Purity/1FRt1SRqqt*FBt/event_purity‚0.00859908014535904 :"Purity/1FRt1SRqqt0FBt/event_purity‚0.00625558290630579 7 Purity/*FRt1SRqqt0FBt/FRt_purity‚0.3561079204082489 6 Purity/*FRt1SRqqt1FBt/FRt_purity‚0.522887110710144 ("Purity/*FRt2SRqqt1FBt/event_purity‚0 ;"Purity/0FRt2SRqqt0FBt/event_purity‚0.055943019688129425 7 Purity/1FRt0SRqqt0FBt/FRt_purity‚0.1023736447095871 7 Purity/1FRt0SRqqt1FBt/FRt_purity‚0.5415266752243042 7 Purity/1FRt1SRqqt2FBt/FRt_purity‚0.7777777910232544 <&Purity/2FRt*SRqqt*FBt/event_proportion‚0.040062315762043 7 Purity/0FRt*SRqqt2FBt/FBt_purity‚0.9319707751274109 ("Purity/0FRt0SRqqt0FBt/event_purity‚1 >&Purity/0FRt2SRqqt*FBt/event_proportion‚0.02457682229578495 9"Purity/0FRt2SRqqt*FBt/SRqqt_purity‚0.2579576075077057 =&Purity/1FRt*SRqqt*FBt/event_proportion‚0.6297432780265808 >&Purity/1FRt*SRqqt1FBt/event_proportion‚0.01643880270421505 ;"Purity/2FRt*SRqqt*FBt/event_purity‚0.006241144146770239 ;"Purity/2FRt*SRqqt0FBt/event_purity‚0.002370529342442751 +"Purity/2FRt1SRqqt1FBt/SRqqt_purity‚0.75 ?&Purity/2FRt2SRqqt*FBt/event_proportion‚0.006417410913854837 ?&Purity/2FRt2SRqqt0FBt/event_proportion‚0.006417410913854837 particle/accuracy_2_of_4‚1 7 Purity/1FRt1SRqqt2FBt/FBt_purity‚0.6388888955116272 =&Purity/2FRt1SRqqt*FBt/event_proportion‚0.0127883180975914 ?&Purity/2FRt1SRqqt0FBt/event_proportion‚0.011928013525903225 +jet/accuracy_2_of_3‚0.26754212379455566 jet/accuracy_1_of_5‚1 & Purity/2FRt0SRqqt1FBt/FBt_purity‚1 7 Purity/0FRt0SRqqt2FBt/FBt_purity‚0.9463458061218262 ("Purity/2FRt1SRqqt0FBt/event_purity‚0 7 Purity/*FRt*SRqqt*FBt/FBt_purity‚0.9752777814865112 :"Purity/0FRt2SRqqt*FBt/event_purity‚0.05401565879583359 :"Purity/1FRt*SRqqt2FBt/event_purity‚0.46774378418922424 8 Purity/1FRt2SRqqt0FBt/FRt_purity‚0.26089149713516235 8 Purity/1FRt2SRqqt1FBt/FRt_purity‚0.10000000149011612 9"Purity/2FRt*SRqqt*FBt/SRqqt_purity‚0.2441549003124237 :"Purity/2FRt*SRqqt0FBt/SRqqt_purity‚0.22837689518928528 9"Purity/0FRt*SRqqt2FBt/SRqqt_purity‚0.4615384638309479 >&Purity/0FRt2SRqqt0FBt/event_proportion‚0.02385602705180645 :"Purity/0FRt2SRqqt0FBt/SRqqt_purity‚0.25990593433380127 ."Purity/0FRt2SRqqt1FBt/SRqqt_purity‚0.21875 9"Purity/0FRt*SRqqt2FBt/event_purity‚0.8403047919273376 =&Purity/0FRt0SRqqt*FBt/event_proportion‚0.0390159972012043 @&Purity/0FRt1SRqqt2FBt/event_proportion‚0.0004417782765813172 ,&Purity/0FRt2SRqqt2FBt/event_proportion‚0 9"Purity/1FRt*SRqqt2FBt/SRqqt_purity‚0.3888888955116272 :"Purity/1FRt1SRqqt*FBt/SRqqt_purity‚0.13301950693130493 :"Purity/1FRt1SRqqt0FBt/SRqqt_purity‚0.11355645209550858 6 Purity/2FRt*SRqqt0FBt/FRt_purity‚0.190482497215271 =&Purity/*FRt0SRqqt0FBt/event_proportion‚0.4607747495174408 =&Purity/*FRt1SRqqt1FBt/event_proportion‚0.0188569575548172 5 Purity/2FRt*SRqqt1FBt/FRt_purity‚0.42208331823349 8 Purity/2FRt0SRqqt*FBt/FRt_purity‚0.11458112299442291 ,&Purity/2FRt2SRqqt1FBt/event_proportion‚0 =&Purity/*FRt*SRqqt1FBt/event_proportion‚0.0617094486951828 9"Purity/*FRt*SRqqt2FBt/event_purity‚0.7327017784118652 ("Purity/*FRt1SRqqt2FBt/event_purity‚0 9"Purity/2FRt1SRqqt0FBt/SRqqt_purity‚0.1361302137374878 9"Purity/0FRt1SRqqt2FBt/SRqqt_purity‚0.4615384638309479 7 Purity/*FRt0SRqqt*FBt/FBt_purity‚0.9789225459098816 , Purity/0FRt2SRqqt1FBt/FBt_purity‚0.96875 7 Purity/2FRt1SRqqt*FBt/FBt_purity‚0.9736841917037964 V÷îÍparticle/accuracy_2_of_5‚1 9"Purity/*FRt1SRqqt2FBt/SRqqt_purity‚0.4444444477558136 9"Purity/*FRt*SRqqt2FBt/SRqqt_purity‚0.4444444477558136 =&Purity/0FRt0SRqqt0FBt/event_proportion‚0.0462472103536129 7 Purity/1FRt*SRqqt*FBt/FBt_purity‚0.9835972785949707ƒíðökÝ‚(:à ûðòÒ¬§ï¤ disk.rbd0.out‚0" disk.nvme0c0n1.in‚ 1852.4453125 network.sent‚ 2.317949e+06( disk./.usagePercent‚76.3247839190482 disk.dm-0.in‚ 1982.2109375& disk.nvme0c0n1.out‚19729.892578125$ disk.nvme0n1.out‚19729.892578125$ memory_percent‚98.81541728973389( proc.memory.availableMB‚ 194.08203125 proc.memory.rssMB‚ 2827.5625& disk.nvme1c1n1.out‚11575.736328125$ disk.nvme1n1.out‚11575.736328125 cpu‚25.818810500372013 proc.cpu.threads‚17 disk.nvme1c1n1.in‚ 129.765625 disk.nvme1n1.in‚ 129.765625" disk.dm-0.out‚30247.4931640625 disk.dm-1.in‚0 disk.nvme0n1.in‚ 1852.4453125* proc.memory.percent‚17.258071899414062 network.recv‚334872% disk./.usageGB‚1171.7762451171875 disk.md0.in‚0 disk.md0.out‚ 29.73828125 disk.dm-1.out‚994.4541015625 disk.rbd0.in‚0Û Ë«Þ‚(:’ ûðòÒ¦Š¡¥ gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚47.0 gpu.0.powerWatts‚113.657' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚37.885666666666665 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0‚È-gß‚(:Ü ŠñòÒ߷Ť$ memory_percent‚98.81210327148438# proc.memory.availableMB‚194.625) disk./.usagePercent‚76.32478640378831 disk.md0.out‚ 30.171875 disk.dm-0.in‚ 1982.2109375# disk.dm-0.out‚30582.01513671875) proc.memory.percent‚17.25454330444336 proc.cpu.threads‚17% disk./.usageGB‚1171.7762832641602 disk.rbd0.out‚0 disk.nvme0n1.in‚ 1852.4453125& disk.nvme0n1.out‚20064.26416015625 disk.dm-1.in‚0! proc.memory.rssMB‚ 2826.984375 network.recv‚339339 disk.rbd0.in‚0 disk.nvme1n1.in‚ 129.765625 disk.md0.in‚0! disk.dm-1.out‚1058.5087890625 cpu‚25.826483678919445 network.sent‚ 2.37508e+06" disk.nvme0c0n1.in‚ 1852.4453125( disk.nvme0c0n1.out‚20064.26416015625 disk.nvme1c1n1.in‚ 129.765625$ disk.nvme1c1n1.out‚ 11640.8671875" disk.nvme1n1.out‚ 11640.8671875 ƒJžà‚(:“ Šñò҉웥 gpu.0.gpu‚76.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚49.0 gpu.0.powerWatts‚223.978' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚74.65933333333334 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0üˆ2á- _wandbruntime‚1137  _runtime‚1137èd÷ÈÉâà ' lr-AdamW/pg2‚0.00013785208224942057 trainer/global_step‚1649 % _timestamp‚1.7844614627620473e+09 _runtime‚1137.150540184 _step‚132 ' lr-AdamW/pg1‚0.00013785208224942057²„oÎãÈ % _timestamp‚1.7844614627620473e+09 _runtime‚1137.150540184 _step‚132 ' lr-AdamW/pg1‚0.00013785208224942057 ' lr-AdamW/pg2‚0.00013785208224942057 trainer/global_step‚1649„`¸Û2ä- _wandbruntime‚1137  _runtime‚1137íôK Ú'åÔO PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚13079023 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1360927178752 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚13079021 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1401798618624 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚129363100661248 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1401683854848 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚129362994961920 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 _runtime‚1137.151404996 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚12521701 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚183 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚128002084822528 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚16591275 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚95885970110976 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 208873472 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚128002067783168 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1357281504298 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚97287862839296 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚95886064220672 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚16601418 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 trainer/global_step‚1649 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚16601418 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚13079021 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚12521518 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚127848850380352 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚128002084822528 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚13079023 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚179 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1357194717872 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚129363100661248 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚127848954206138 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚8776721 _step‚133 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚129362994961920 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1360927178752 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 % _timestamp‚1.7844614627629285e+09 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚29670296 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚126491655662480 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚16591275 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚8776717 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚3744801 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚97287653965824 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚128002067783168 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 114763776 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚3744980 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0ªÀ²¬ß'æÙO QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚13079021 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1401798618624 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚129363100661248 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1401683854848 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚129362994961920 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 _runtime‚1137.151404996 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚12521701 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚183 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚128002084822528 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚16591275 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 `MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚95885970110976 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 208873472 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚128002067783168 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1357281504298 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 ]JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚97287862839296 dQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚95886064220672 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚16601418 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 trainer/global_step‚1649 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚16601418 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚13079021 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚12521518 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚127848850380352 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚128002084822528 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚13079023 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚179 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1357194717872 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚129363100661248 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚127848954206138 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚8776721 _step‚133 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚129362994961920 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1360927178752 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 % _timestamp‚1.7844614627629285e+09 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚29670296 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚126491655662480 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚16591275 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚8776717 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚3744801 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 YFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚97287653965824 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚128002067783168 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 114763776 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚3744980 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚13079023 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1360927178752 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0…f!‘2ç- _wandbruntime‚1137  _runtime‚1137£x¼Y£"èÝM REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚127924028226242 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚126565963227232 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚ 1137.57167224 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚128077234488832 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚128077217449472 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚128077234488832 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1361801561600 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372523008 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 116853760 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚95942201948672 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚13086652 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1361801561600 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚29687745 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 _step‚134 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚12528933 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚16601095 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚13086652 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚12528748 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚29697904 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1402609238016 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚126565946187872 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚8781773 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚3747156 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚127923924400344 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚128077217449472 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚185 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚8781777 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚129438930343936 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚129439036050432 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 K>DeviceStatsMonitor.on_train_batch_end/a’Ä@llocation.all.allocated‚29697904 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 210963456 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚95942107838976 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚16611252 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1357978212472 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚13086650 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 ²Vè&éâM C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚16611252 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚12528748 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚29697904 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1402609238016 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚126565946187872 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚8781773 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚3747156 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚127923924400344 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚128077217449472 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚185 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚8781777 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚129438930343936 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚129439036050432 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚29697904 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 210963456 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚95942107838976 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚16611252 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1357978212472 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚13086650 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚127924028226242 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚126565963227232 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚ 1137.57167224 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚128077234488832 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚128077217449472 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚128077234488832 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1361801561600 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372523008 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 116853760 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚95942201948672 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚13086652 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1361801561600 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚29687745 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 _step‚134 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚12528933 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚16601095 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚13086652 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 FDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚9029517 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1442446152192 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1400365778944 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚133159828217344 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚3853578 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1442333485568 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚133159933916672 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚9029517 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1442446152192 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1400365778944 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚133159828217344 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚3853578 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1442333485568 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚133159933916672 I8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚30572206 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 _runtime‚1171.508368219 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372523008 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚131677793026650 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1401062834688 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 208866304 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚9034573 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚17104104 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚30572206 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 trainer/global_step‚1699 % _timestamp‚1.7844614971197422e+09 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1397216743280 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚13468100 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚131677689200752 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚177 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _step‚138 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚131834717804032 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚17104104 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚12890450 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚98748994238976 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 114756608 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚17093947 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1401062834688 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚133235763599360 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚12890269 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0Ü{šë&ûåM XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1401062834688 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚133235763599360 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚12890269 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 bODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚98749088348672 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1397303529818 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3855696 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚130280489496832 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚100192140472832 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚3855873 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚100192349339136 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚30572206 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 _runtime‚1171.508368219 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372523008 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚131677793026650 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1401062834688 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 208866304 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚9034573 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚17104104 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚30572206 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 trainer/global_step‚1699 % _timestamp‚1.7844614971197422e+09 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1397216743280 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚13468100 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚131677689200752 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚177 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _step‚138 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚131834717804032 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚17104104 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚12890450 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 ^KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚98748994238976 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 114756608 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚17093947 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812Š\Y5ù2ü- _wandbruntime‚1171  _runtime‚1171ÇMüGý‚ 2loss/SRqqt2/assignment_loss‚0.9159082736403601 1loss/FBt2/assignment_loss‚0.24280362019008242 1loss/SRqqt1/detection_loss‚1.8344159729760396 _runtime‚1171.512810747 0loss/FRt1/assignment_loss‚3.2009596581375876 0loss/FRt2/assignment_loss‚1.6535517215350917 2loss/SRqqt1/assignment_loss‚1.1474116174636577 1loss/SRqqt2/detection_loss‚0.7523869631134197 /loss/FBt2/detection_loss‚1.1832318044675456 % _timestamp‚1.7844614971209307e+09 _step‚139 /loss/FRt1/detection_loss‚0.6283200096152182 /loss/FRt2/detection_loss‚1.3618427755300049 /loss/FBt1/detection_loss‚1.9211952021097543 epoch‚4 trainer/global_step‚1699 1loss/FBt1/assignment_loss‚0.12414957305706048 &loss/total_loss‚14.966177191835822Ÿ‚lÜ þ‡ 0loss/FRt1/assignment_loss‚3.2009596581375876 0loss/FRt2/assignment_loss‚1.6535517215350917 2loss/SRqqt1/assignment_loss‚1.1474116174636577 1loss/SRqqt2/detection_loss‚0.7523869631134197 /loss/FBt2/detection_loss‚1.1832318044675456 % _timestamp‚1.7844614971209307e+09 _step‚139 /loss/FRt1/detection_loss‚0.6283200096152182 /loss/FRt2/detection_loss‚1.3618427755300049 /loss/FBt1/detection_loss‚1.9211952021097543 epoch‚4 trainer/global_step‚1699 1loss/FBt1/assignment_loss‚0.12414957305706048 &loss/total_loss‚14.966177191835822 2loss/SRqqt2/assignment_loss‚0.9159082736403601 1loss/FBt2/assignment_loss‚0.24280362019008242 1loss/SRqqt1/detection_loss‚1.8344159729760396 _runtime‚1171.512810747‹ˆ_—tÿ‚(:é ÆñòÒÑÛ¤% disk./.usageGB‚1171.7752418518066 disk.dm-1.in‚0" disk.dm-1.out‚1059.27294921875 disk.nvme0n1.in‚ 1852.4453125& disk.nvme0n1.out‚20075.59423828125( disk.nvme1c1n1.out‚11644.57373046875 disk.md0.out‚31.5625) proc.memory.percent‚17.25454330444336 network.recv‚352778 disk.rbd0.in‚0 disk.nvme1c1n1.in‚ 129.765625 disk.dm-0.in‚ 1982.2109375$ memory_percent‚98.81412982940674 cpu‚25.824732761510344 proc.cpu.threads‚17 network.sent‚ 2.451226e+06) disk./.usagePercent‚76.32471857038311( disk.nvme0c0n1.out‚20075.59423828125& disk.nvme1n1.out‚11644.57373046875 disk.md0.in‚0# disk.dm-0.out‚30593.27783203125! proc.memory.rssMB‚ 2826.984375 disk.rbd0.out‚0" disk.nvme0c0n1.in‚ 1852.4453125 disk.nvme1n1.in‚ 129.765625( proc.memory.availableMB‚ 194.29296875;‚€ ‚(:’ ÆñòÒ­¸È¤ gpu.0.gpu‚37.0 gpu.0.memory‚8, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚48.0 gpu.0.powerWatts‚162.382' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚54.12733333333334 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0™4cœ ‚(:‘ ÕñòÒ¨ôŤ gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚48.0 gpu.0.powerWatts‚207.073' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚69.02433333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0^MJj‚ ‚(:ß ÕñòÒǪפ' proc.memory.availableMB‚ 193.7109375 cpu‚25.821956222158683 network.sent‚ 2.515866e+06 network.recv‚356770$ disk.nvme1n1.out‚11645.123046875 disk.rbd0.out‚0( disk.nvme0c0n1.out‚20076.43212890625 proc.memory.rssMB‚ 2827.5625& disk.nvme0n1.out‚20076.43212890625 disk.nvme1c1n1.in‚ 129.765625 disk.nvme1n1.in‚ 129.765625 disk.md0.in‚0 disk.dm-0.in‚ 1982.2109375 disk.dm-1.out‚ 1059.36328125% disk./.usageGB‚1171.7752113342285 disk.nvme0n1.in‚ 1852.4453125& disk.nvme1c1n1.out‚11645.123046875# disk.dm-0.out‚30593.89892578125" disk.nvme0c0n1.in‚ 1852.4453125* proc.memory.percent‚17.258071899414062 proc.cpu.threads‚17) disk./.usagePercent‚76.32471658259101 disk.md0.out‚31.875 disk.dm-1.in‚0 disk.rbd0.in‚0$ memory_percent‚98.81768226623535R,ÌV2ƒ - _wandbruntime‚1205  _runtime‚1205ÈW?-É„ à _runtime‚1205.079575923 _step‚140 ' lr-AdamW/pg1‚0.00013762141190832794 ' lr-AdamW/pg2‚0.00013762141190832794 trainer/global_step‚1749 % _timestamp‚1.7844615306911786e+09f·W©Î… È _runtime‚1205.079575923 _step‚140 ' lr-AdamW/pg1‚0.00013762141190832794 ' lr-AdamW/pg2‚0.00013762141190832794 trainer/global_step‚1749 % _timestamp‚1.7844615306911786e+09Œεg‰2† - _wandbruntime‚1205  _runtime‚12056ÚM÷” ‡ ×O [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚135517051452928 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚135517034413568 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1439627059200 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚135517051452928 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1439627059200 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372523008 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1435758565914 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚135517034413568 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 trainer/global_step‚1749 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚101499837020672 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚135356379981168 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚3962402 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1439715719168 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚13841923 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚13841923 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚136956661472768 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 _step‚141 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1483080996352 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚17587122 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚13244723 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚133920708201680 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚13841921 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚133920725241040 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 217262080 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚135356483806954 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚194 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚9282321 % _timestamp‚1.7844615306922147e+09 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚136956661472768 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 120650260487ëâ'ˆ ÜO \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚135517034413568 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 trainer/global_step‚1749 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚101499837020672 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚135356379981168 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚3962402 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1439715719168 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚13841923 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚13841923 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚136956661472768 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 _step‚141 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1483080996352 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚17587122 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚13244723 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚133920708201680 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚13841921 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚133920725241040 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 217262080 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚135356483806954 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚194 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚9282321 % _timestamp‚1.7844615306922147e+09 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚136956661472768 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚135517051452928 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚135517034413568 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1439627059200 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚135517051452928 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1439627059200 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372523008 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1435758565914 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812+ê22‰ - _wandbruntime‚1205  _runtime‚1205ÍèÃè&Š âM 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚199 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚135592184079872 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1440412774912 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚135431557827058 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚13251978 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚101555974748672 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1440501442048 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚13251779 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚31446508 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚17586799 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚17586799 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚133995015766432 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚9287373 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚135592201119232 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚13849552 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚13849550 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3964406 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 225643520 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372523008 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚135431454001160 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚101555880638976 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚17596956 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚133994998727072 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚195 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1436455274088 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _runtime‚1205.498256371 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚31446508:ß3ܘ‹ çM XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1440412774912 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚103039645439488 trainer/global_step‚1749 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚137032596854784 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚13849550 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1483764800512 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚17586799 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚133995015766432 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚9287373 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚135592201119232 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚13849552 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚13849550 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚3964406 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 225643520 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372523008 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚135431454001160 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚101555880638976 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚17596956 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚133994998727072 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚195 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1436455274088 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _runtime‚1205.498256371 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚31446508 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 H@DeviceStatsMonitor.on_train_batch_end/segment.large_S÷>_Upool.current‚682 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚199 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚135592184079872 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1440412774912 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚135431557827058 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚13251978 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚101555974748672 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1440501442048 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚13251779 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚31446508 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚17586799 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 E–“ ‚(:ˆ äñòÒÒ¡Î¥ gpu.0.gpu‚53.0 gpu.0.memory‚13, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚47.0 gpu.0.powerWatts‚113.718' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚37.906 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0j{ s‘ ‚(:è óñòÒ·Èí¤ disk.nvme0n1.in‚ 1852.4453125% disk.nvme0n1.out‚20096.7998046875( disk.nvme1c1n1.out‚11648.76025390625& disk.nvme1n1.out‚11648.76025390625 disk.dm-0.in‚ 1982.2109375 disk.md0.out‚ 32.69140625' disk.nvme0c0n1.out‚20096.7998046875 proc.cpu.threads‚17 network.recv‚364357 disk.nvme1c1n1.in‚ 129.765625 disk.nvme1n1.in‚ 129.765625" disk.dm-0.out‚30615.8876953125 disk.dm-1.in‚0# memory_percent‚98.8166093826294( proc.memory.availableMB‚ 193.88671875 proc.memory.rssMB‚ 2827.5625) disk./.usagePercent‚76.27826784472768 disk.md0.in‚0" disk.dm-1.out‚1059.63720703125 disk.rbd0.in‚0 disk.rbd0.out‚0" disk.nvme0c0n1.in‚ 1852.4453125* proc.memory.percent‚17.258071899414062 cpu‚25.816593680099874 network.sent‚ 2.583144e+06% disk./.usageGB‚1171.0621070861816í@ïž’ ‚(:“ óñòÒ…›¥ gpu.0.gpu‚90.0 gpu.0.memory‚16, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚49.0 gpu.0.powerWatts‚213.121' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚71.04033333333334 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0[ V2“ - _wandbruntime‚1238  _runtime‚1238õÿÞ@É” à trainer/global_step‚1799 % _timestamp‚1.7844615640602846e+09 _runtime‚1238.449801866 _step‚144 ' lr-AdamW/pg1‚0.00013749952168301866 ' lr-AdamW/pg2‚0.00013749952168301866›=æΕ È trainer/global_step‚1799 % _timestamp‚1.7844615640602846e+09 _runtime‚1238.449801866 _step‚144 ' lr-AdamW/pg1‚0.00013749952168301866 ' lr-AdamW/pg2‚0.00013749952168301866ƒÂ ‰2– - _wandbruntime‚1238  _runtime‚1238xÅmÞ'— ØO GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1478976999424 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚105830409383424 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚139110248607362 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚18069831 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚32303347 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 _runtime‚1238.449891716 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚137635234471280 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚139274517728768 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚13606387 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1474997096722 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1479065659392 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚32293202 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1479065659392 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚178 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1474910310296 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚14223373 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1478976999424 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚140753494728192 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚139274517728768 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚140753600427520 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚4071266 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚9535117 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚104306723420672 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 % _timestamp‚1.7844615640610552e+09 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚140753494728192 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚139110144781576 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚137635251510640 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚14223371 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚9535121 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1474997096722 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1479065659392 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚32293202 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1479065659392 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚178 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1474910310296 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚14223373 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1478976999424 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚140753494728192 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚139274517728768 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚140753600427520 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚4071266 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚9535117 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚104306723420672 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 % _timestamp‚1.7844615640610552e+09 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚140753494728192 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚139110144781576 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚137635251510640 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚14223371 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚9535121 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚32320810 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚18089808 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚176 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚105887351857152 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 _step‚146 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚18079651 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1479762715136 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚13613376 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 _runtime‚1238.867160569 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 EDeviceStat¯¨º¬ãsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚14231002 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 116853760 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚137709542036032 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚13613552 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚137709524996672 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚139349684434432 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚139185218801568 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚18079651 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚140829535816704 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1479762715136 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1475780591434 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1475693804896 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚14231002 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚32320810 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚9540173 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚14231000 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 210963456 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚105887140893696 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚32310651 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚104362767038976 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚9540177 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1479851382272 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚4073203 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1524373854720 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚32310651 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚14231000 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1524490708480 $ _timestamp‚1.784461564477864e+09 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚18089808 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚4073375 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372523008 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚104362861148672 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚140829430110208 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682/Æãbì&› æM WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚139349667395072 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1479851382272 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚140829535816704 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚14231002 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 116853760 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚137709542036032 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚13613552 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚137709524996672 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚139349684434432 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚139185218801568 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚18079651 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚140829535816704 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1479762715136 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1475780591434 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1475693804896 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚14231002 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚32320810 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚9540173 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚14231000 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 210963456 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚105887140893696 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚32310651 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚104362767038976 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚9540177 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1479851382272 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚4073203 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1524373854720 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚32310651 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚14231000 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1524490708480 $ _timestamp‚1.784461564477864e+09 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚18089808 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚4073375 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372523008 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚104362861148672 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚140829430110208 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚32320810 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚18089808 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚176 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚105887351857152 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 _step‚146 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚18079651 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1479762715136 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚13613376 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 _runtime‚1238.867160569 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 EDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚144550433682944 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚13965987 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 225650688 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚107113609820672 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1518326939648 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 _runtime‚1271.605022762 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚14604821 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚143032018083328 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1514235627530 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1518415599616 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚14604821 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚142863909581984 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1564171592192 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1564040051200 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚144550327983616 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 trainer/global_step‚1849 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 % _timestamp‚1.7844615972167573e+09 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚143032001043968 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚18572826 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚197 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚13965790 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚193 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚142864013407770 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚4177873 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚108677555762176 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚9787921 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372523008 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚18562683 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚33177649 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚107113515710976 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚33167504 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0R]¦¨ ÝO YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚142864013407770 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚4177873 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚108677555762176 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚9787921 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372523008 60DeviceStatsMonitor.on_trainúV•¬=$_batch_start/num_ooms‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚18562683 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚33177649 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚107113515710976 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚33167504 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚144550433682944 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚143032018083328 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚18562683 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1514148841104 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚144550433682944 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚13965987 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 225650688 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚107113609820672 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1518326939648 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 _runtime‚1271.605022762 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚14604821 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚143032018083328 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1514235627530 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1518415599616 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚14604821 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚142863909581984 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1564171592192 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1564040051200 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚144550327983616 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 trainer/global_step‚1849 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 % _timestamp‚1.7844615972167573e+09 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚143032001043968 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚18572826 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚197 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚13965790 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚193•?„b2© - _wandbruntime‚1272  _runtime‚1272®«u[è&ª âM S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚144626263365632 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1514932335704 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚18572503 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚108734716186112 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚143107150710272 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚14612452 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1519201322496 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚18582660 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚14612450 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚33184953 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚18582660 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1519112655360 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚143107167749632 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚4179890 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚33184953 % _timestamp‚1.7844615976407056e+09 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚33195112 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚13972863 trainer/global_step‚1849 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚33195112 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1564968637440 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372523008 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚173 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1519112655360 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚18572503 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚144626369072128 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚141424068305632 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚14612450 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚142938983601976 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚4180059 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812˜ÇºØí&« çM WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚144626369072128 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚143107150710272 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚108734507319808 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 114756608 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1564853880832 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚141424068305632 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚14612450 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚142938983601976 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚4180059 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚144626263365632 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1514932335704 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚18572503 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚108734716186112 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚143107150710272 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚14612452 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1519201322496 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚18582660 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚14612450 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚33184953 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚18582660 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1519112655360 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚143107167749632 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚4179890 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚33184953 % _timestamp‚1.7844615976407056e+09 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚33195112 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚13972863 trainer/global_step‚1849 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚33195112 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1564968637440 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372523008 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚173 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1519112655360 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚18572503 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚144626369072128 PDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚4284659 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 125249536 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 _runtime‚1305.817864686 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 % _timestamp‚1.7844616314259596e+09 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚111525176754688 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚14986273 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚109920496220672 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1557765539840 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚14986273 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372523008 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚146789501398528 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚148347266938368 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚146789484359168 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚145064304049840 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚19055535 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚148347161239040 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1557676879872 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚14986271 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1604555284480 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1553474158338 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚19065678 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚146789501398528 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1553387371912 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1557765539840 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚34051951 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚109920402110976 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚10040717 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1557676879872 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 _step‚1531Ã#Îå'º ßO =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚111525176754688 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚14986273 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚109920496220672 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1557765539840 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚14986273 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372523008 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚146789501398528 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚148347266938368 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚146789484359168 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚145064304049840 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚19055535 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚148347161239040 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1557676879872 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚14986271 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1604555284480 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1553474158338 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚19065678 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚146789501398528 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1553387371912 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1557765539840 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚34051951 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚109920402110976 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚10040717 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1557676879872 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 _step‚153 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 trainer/global_step‚1899 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚14986271 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚148347266938368 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚14325380 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚4284492 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚4284659 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 125249536 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 _runtime‚1305.817864686 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 % _timestamp‚1.7844616314259596e+09™ 1Ð¥2» - _wandbruntime‚1306  _runtime‚1306íÌŠ&ä%¼ äM UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚111582130770944 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚14332379 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1605363191296 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 _step‚154 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚148423096621056 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚146692748402384 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚178 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1558551262720 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚174 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 133630976 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚146864634025472 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚14332557 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚34069414 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚109976633948672 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚109976539838976 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1558551262720 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚34059255 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚19065355 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚146864651064832 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚146864651064832 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚146864634025472 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚34069414 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚19065355 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚148423202327552 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚10045773 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 TFDeviceStatsMonitor.on_train_batâǽch_end/inactive_split_bytes.all.current‚ 227740672 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 trainer/global_step‚1899 % _timestamp‚1.7844616318486526e+09h‰hßï&½ éM LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚174 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 133630976 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚146864634025472 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚14332557 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚34069414 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚109976633948672 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚109976539838976 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1558551262720 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚34059255 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚19065355 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚146864651064832 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚146864651064832 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚146864634025472 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚34069414 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚19065355 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚148423202327552 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚10045773 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 227740672 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 trainer/global_step‚1899 % _timestamp‚1.7844616318486526e+09 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚111582130770944 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚14332379 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1605363191296 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 _step‚154 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚148423096621056 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚146692748402384 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚178 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1558551262720š‘¨{z2¾ - _wandbruntime‚1306  _runtime‚1306Ö뽿 þ % _timestamp‚1.7844616318498275e+09 _runtime‚1306.240850611 0loss/FBt1/assignment_loss‚0.1465093380313867 1loss/SRqqt2/detection_loss‚0.7905006254628331 &loss/total_loss‚15.718973306219489 2loss/SRqqt1/assignment_loss‚1.0740568730736355 1loss/FBt2/assignment_loss‚0.33057625961422066 /loss/FRt1/detection_loss‚0.5756235700035685 1loss/SRqqt1/detection_loss‚1.8974651776702773 _step‚155 /loss/FBt2/detection_loss‚2.0888493293275436 epoch‚4 /loss/FRt1/assignment_loss‚2.996854107463083 0loss/FRt2/assignment_loss‚1.5536166428490212 2loss/SRqqt2/assignment_loss‚0.8714095608816302 /loss/FRt2/detection_loss‚1.4503575428069304 -loss/FBt1/detection_loss‚1.94315427903536 trainer/global_step‚1899Ãtuz À ƒ /loss/FBt2/detection_loss‚2.0888493293275436 epoch‚4 /loss/FRt1/assignment_loss‚2.996854107463083 0loss/FRt2/assignment_loss‚1.5536166428490212 2loss/SRqqt2/assignment_loss‚0.8714095608816302 /loss/FRt2/detection_loss‚1.4503575428069304 -loss/FBt1/detection_loss‚1.94315427903536 trainer/global_step‚1899 % _timestamp‚1.7844616318498275e+09 _runtime‚1306.240850611 0loss/FBt1/assignment_loss‚0.1465093380313867 1loss/SRqqt2/detection_loss‚0.7905006254628331 &loss/total_loss‚15.718973306219489 2loss/SRqqt1/assignment_loss‚1.0740568730736355 1loss/FBt2/assignment_loss‚0.33057625961422066 /loss/FRt1/detection_loss‚0.5756235700035685 1loss/SRqqt1/detection_loss‚1.8974651776702773 _step‚155›C˜®¢^Á ‚(:Ó ÍòòÒ¾›ý¤$ memory_percent‚98.81792068481445 network.sent‚ 2.739412e+06$ disk./.usageGB‚1171.833667755127 disk.rbd0.in‚0 proc.memory.rssMB‚ 2827.5625" disk.dm-1.out‚1060.81787109375 network.recv‚386441) disk./.usagePercent‚76.32852419834731 disk.md0.in‚0* proc.memory.percent‚17.258071899414062 cpu‚25.810219139384007 proc.cpu.threads‚17& disk.nvme1n1.out‚11906.77099609375 disk.md0.out‚ 34.90234375 disk.dm-0.in‚ 1982.2109375" disk.nvme0c0n1.in‚ 1852.4453125 disk.nvme0n1.in‚ 1852.4453125 disk.nvme0n1.out‚20681.75& proc.memory.availableMB‚ 193.671875 disk.nvme1c1n1.in‚ 129.765625( disk.nvme1c1n1.out‚11906.77099609375 disk.nvme1n1.in‚ 129.765625 disk.dm-0.out‚ 31452.8984375 disk.dm-1.in‚0 disk.rbd0.out‚0 disk.nvme0c0n1.out‚20681.75Р‰N‘ ‚(:† ÍòòҨ˯¥ gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚48.0 gpu.0.powerWatts‚219.306' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚73.102 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Í©0¯rà ‚(:ç ÜòòÒ÷øÕ¤& disk.nvme1n1.out‚11907.48193359375 disk.md0.in‚0 disk.md0.out‚ 35.2421875" disk.dm-0.out‚31454.7744140625' disk.nvme0c0n1.out‚20683.7197265625( disk./.usagePercent‚76.3285219620812 disk.nvme1c1n1.in‚ 129.765625 disk.dm-0.in‚ 1982.2109375" disk.nvme0c0n1.in‚ 1852.4453125$ memory_percent‚98.81751537322998 proc.memory.rssMB‚ 2827.5625 cpu‚25.816235941101265 proc.cpu.threads‚17 network.sent‚ 2.787909e+06 network.recv‚390706% disk./.usageGB‚1171.8336334228516% disk.nvme0n1.out‚20683.7197265625 disk.rbd0.in‚0( proc.memory.availableMB‚ 193.73828125* proc.memory.percent‚17.258071899414062 disk.dm-1.in‚0" disk.dm-1.out‚1060.87255859375 disk.rbd0.out‚0 disk.nvme0n1.in‚ 1852.4453125( disk.nvme1c1n1.out‚11907.48193359375 disk.nvme1n1.in‚ 129.765625‘¢€‚Ä ‚(:’ ÜòòÒÓ𞥠gpu.0.gpu‚28.0 gpu.0.memory‚6, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚48.0 gpu.0.powerWatts‚221.925' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚73.97500000000001 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0UR ÷2Å - _wandbruntime‚1340  _runtime‚1340$v„:ÉÆ Ã % _timestamp‚1.7844616661629925e+09 _runtime‚1340.551353409 _step‚156 ' lr-AdamW/pg1‚0.00013710772557193367 ' lr-AdamW/pg2‚0.00013710772557193367 trainer/global_step‚1949Z6FùÎÇ È _step‚156 ' lr-AdamW/pg1‚0.00013710772557193367 ' lr-AdamW/pg2‚0.00013710772557193367 trainer/global_step‚1949 % _timestamp‚1.7844616661629925e+09 _runtime‚1340.551353409œצA‰2È - _wandbruntime‚1340  _runtime‚1340A ½à'É ÚO GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚152143994494464 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚15367721 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚15367721 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚4389867 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚34916108 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1597115480064 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚10293517 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 131540992 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚148778813280080 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚19558530 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1597026820096 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚14683565 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1592625902720 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚10293521 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚19558530 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚177 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚152143994494464 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚150546967674368 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚19548387 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚148778830319440 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚150546984713728 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚4390044 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚19548387 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚150546984713728 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚14683384 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 % _timestamp‚1.7844616661640587e+09 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚150546967674368 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 _step‚157 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1597115480064 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372523008 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1645083436032 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚150371543008586 _runtime‚1340.552470171 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚150371543008586 _runtime‚1340.552470171 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚152143994494464 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚150546967674368 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚19548387 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚148778830319440 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚150546984713728 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚4390044 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚19548387 ]IDeviceStatsMonitor.on_train_batch_start/actiίb[ ve_bytes.large_pool.allocated‚150546984713728 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚14683384 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 % _timestamp‚1.7844616661640587e+09 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚150546967674368 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 _step‚157 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1597115480064 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372523008 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1645083436032 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 IDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1597901202944 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚4392018 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1645893693952 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚10298573 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚19558207 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚150446617028690 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 _runtime‚1340.971844036 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚150622117340672 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚34933557 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚15375350 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚150446513202792 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚15375352 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚19568364 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚14690415 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 221449216 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1597812535808 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚114429319932928 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚180 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚10298577 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1597901202944 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚148853103805472 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1593496183858 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372523008 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚112783520348672 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚34943716 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚34943716 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚148853120844832 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚150622134380032 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚176 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚15375350 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚19568364 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1597812535808 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1593409397320 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0ŒKdfï&Í éM ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚150622117340672 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚14690595 trainer/global_step‚1949 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1597901202944 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚4392018 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1645893693952 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚10298573 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚19558207 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚150446617028690 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 _runtime‚1340.971844036 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚150622117340672 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚34933557 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚15375350 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚150446513202792 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚15375352 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚19568364 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚14690415 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 221449216 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1597812535808 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚114429319932928 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚180 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚10298577 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1597901202944 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚148853103805472 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1593496183858 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372523008 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚112783520348672 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚34943716 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚34943716 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚148853120844832 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚150622134380032 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚176 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚15375350 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚19568364 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1597812535808 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1593409397320 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0ž?Hs2Î - _wandbruntime‚1340  _runtime‚1340%‚»Ï ÿ /loss/FRt1/detection_loss‚0.5626651695599801 0loss/SRqqt1/detection_loss‚1.883229620974933 /loss/FBt1/detection_loss‚1.4712363269131956 /loss/FRt1/assignment_loss‚3.025685922573033 /loss/FRt2/assignment_loss‚1.595037343556674 2loss/SRqqt2/assignment_loss‚0.9432148633563089 /loss/FRt2/detection_loss‚1.4099020854466557 1loss/SRqqt2/detection_loss‚0.7473172485835767 /loss/FBt2/detection_loss‚2.7607684930068594 &loss/total_loss‚15.597892430714136 % _timestamp‚1.7844616665843725e+09 2loss/SRqqt1/assignment_loss‚0.9854437857453385 epoch‚4 _step‚159 1loss/FBt2/assignment_loss‚0.11098010034555728 trainer/global_step‚1949 _runtime‚1340.972455137 1loss/FBt1/assignment_loss‚0.10241147065202513Z\2 Ð „ _runtime‚1340.972455137 1loss/FBt1/assignment_loss‚0.10241147065202513 /loss/FRt1/detection_loss‚0.5626651695599801 0loss/SRqqt1/detection_loss‚1.883229620974933 /loss/FBt1/detection_loss‚1.4712363269131956 /loss/FRt1/assignment_loss‚3.025685922573033 /loss/FRt2/assignment_loss‚1.595037343556674 2loss/SRqqt2/assignment_loss‚0.9432148633563089 /loss/FRt2/detection_loss‚1.4099020854466557 1loss/SRqqt2/detection_loss‚0.7473172485835767 /loss/FBt2/detection_loss‚2.7607684930068594 &loss/total_loss‚15.597892430714136 % _timestamp‚1.7844616665843725e+09 2loss/SRqqt1/assignment_loss‚0.9854437857453385 epoch‚4 _step‚159 1loss/FBt2/assignment_loss‚0.11098010034555728 trainer/global_step‚1949ŸèŽ|©qÑ ‚(:æ ëòòÒØÞߤ& disk.nvme1n1.out‚11908.36474609375$ disk./.usageGB‚1171.834072113037 disk.rbd0.out‚0" disk.nvme0c0n1.in‚ 1852.4453125 disk.nvme1c1n1.in‚ 129.765625 disk.md0.in‚0" disk.dm-0.out‚31455.5478515625 cpu‚25.82045491408773 network.recv‚393750) disk./.usagePercent‚76.32855053659256 disk.md0.out‚ 35.62109375 disk.dm-0.in‚ 1982.2109375 disk.dm-1.in‚0" disk.dm-1.out‚1061.03271484375 proc.cpu.threads‚17 network.sent‚ 2.793801e+06' disk.nvme0c0n1.out‚20684.5986328125% disk.nvme0n1.out‚20684.5986328125 disk.nvme1n1.in‚ 129.765625( proc.memory.availableMB‚ 129.07421875 proc.memory.rssMB‚ 2827.5625 disk.rbd0.in‚0 disk.nvme0n1.in‚ 1852.4453125( disk.nvme1c1n1.out‚11908.36474609375# memory_percent‚99.2121934890747* proc.memory.percent‚17.258071899414062Ôw(ãÒ ‚(:’ ëòòÒÑŠø¤ gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚47.0 gpu.0.powerWatts‚113.567' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚37.855666666666664 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ü‡¿,uÓ ‚(:ê úòòÒ×ÌÙ¤% disk./.usageGB‚1171.8339805603027 disk.md0.in‚0 disk.md0.out‚ 36.0390625 disk.dm-1.in‚0 proc.memory.rssMB‚ 2827.5625* proc.memory.percent‚17.258071899414062 network.recv‚398096) disk./.usagePercent‚76.32854457321626& disk.nvme1n1.out‚11909.14501953125 disk.rbd0.in‚0 disk.nvme1c1n1.in‚ 129.765625$ memory_percent‚99.21088218688965 proc.cpu.threads‚17 disk.dm-0.in‚ 1982.2109375# disk.dm-0.out‚31458.10400390625" disk.dm-1.out‚1061.07177734375" disk.nvme0c0n1.in‚ 1852.4453125 cpu‚25.821685067583857 disk.nvme1n1.in‚ 129.765625( disk.nvme0c0n1.out‚20687.31787109375 disk.nvme0n1.in‚ 1852.4453125& disk.nvme0n1.out‚20687.31787109375( disk.nvme1c1n1.out‚11909.14501953125' proc.memory.availableMB‚ 129.2890625 disk.rbd0.out‚0 network.sent‚ 2.855332e+06_ýïŸÔ ‚(:” úòòÒÚµ‰¥ gpu.0.gpu‚91.0 gpu.0.memory‚22, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚47.0 gpu.0.powerWatts‚114.152' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚38.050666666666665 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Hpv2Õ - _wandbruntime‚1374  _runtime‚1374—wÉÖ Ã % _timestamp‚1.7844616997355568e+09 _runtime‚1374.123982932 _step‚160 ' lr-AdamW/pg1‚0.00013696844635028183 ' lr-AdamW/pg2‚0.00013696844635028183 trainer/global_step‚1999GC@œÎ× È _runtime‚1374.123982932 _step‚160 ' lr-AdamW/pg1‚0.00013696844635028183 ' lr-AdamW/pg2‚0.00013696844635028183 trainer/global_step‚1999 % _timestamp‚1.7844616997355568e+09 Ê„M2Ø - _wandbruntime‚1374  _runtime‚1374F¤"`ÌÙ ÚO ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1631864433528 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚154125307808994 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372523008 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 trainer/global_step‚1999 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚35790410 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚35800555 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚10546317 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1636376760320 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚15749173 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚155940933449216 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚15749171 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1636465420288 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚117219959354880 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚15042982 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚179 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚15749171 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚155940933449216 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 223553536 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚20041239 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚154304468028928 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1685560890368 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚154304450989568 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚4496486 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1685690334208 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 L?DeviceStatsMonitor.on_train_batch_start/_DEÁactive.small_pool.freed‚20041239 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚117219735801344 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 % _timestamp‚1.7844616997376513e+09 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚154304468028928 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 _runtime‚1374.128961551 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚20051382 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚4496661 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚115534269020672 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 129443840 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚154125203983208 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚155940827749888 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚154304450989568 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1636376760320 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚115534174910976 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚15042803 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚10546321 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚152493339549680w Þå'Ú ßO TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 _runtime‚1374.128961551 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚20051382 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚4496661 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚115534269020672 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 129443840 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚154125203983208 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚155940827749888 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚154304450989568 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1636376760320 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚115534174910976 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚15042803 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚10546321 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚152493339549680 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1631864433528 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚154125307808994 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372523008 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 trainer/global_step‚1999 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚35790410 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚35800555 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚10546317 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1636376760320 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚15749173 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚155940933449216 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚15749171 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1636465420288 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚117219959354880 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚15042982 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚179 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚15749171 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚155940933449216 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 223553536 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚20041239 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚154304468028928 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1685560890368 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚154304450989568 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚4496486 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1685690334208 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚20041239 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚117219735801344 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 % _timestamp‚1.7844616997376513e+09 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚154304468028928¡¾óÔB2Û - _wandbruntime‚1374  _runtime‚1374ôXê&Ü äM NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚20051059 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚154379600655872 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1637162476032 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚15049906 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚15756802 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚117276678388736 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚15756802 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚15756800 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚156016763131904 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚175 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚35807859 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 trainer/global_step‚1999 _step‚162 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚154379617695232 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚20061216 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚179 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚156016868838400 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚152567647114432 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚35807859 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚4498708 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚115590406748672 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚152567630075072 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1632734714666 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1632647928128 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 133630976 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚154200381829098 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚156016868838400 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚15756800 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚35818018 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 _runtime‚1374.529938019 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1686365749760 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1637162476032 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚117276906129408 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372523008 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1637251143168 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚4498533 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚154379600655872 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚10551373 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚154200278003200 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0yúÂÈÝ éM UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1637162476032 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚117276906129408 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372523008 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1637251143168 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚4498533 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚154379600655872 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚10551373 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚154200278003200 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚20051059 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚154379600655872 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1637162476032 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚15049906 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚15756802 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚117276678388736 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚15756802 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚15756800 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚156016763131904 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚175 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚35807859 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 trainer/global_step‚1999 _step‚162 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚154379617695232 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚20061216 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚179 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚156016868838400 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚152567647114432 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚35807859 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚4498708 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚115590406748672 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚152567630075072 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1632734714666 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1632647928128 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 Ešä6' =DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 133630976 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚154200381829098 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚156016868838400 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚15756800 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚35818018 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 _runtime‚1374.529938019 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1686365749760 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279¢"j2Þ - _wandbruntime‚1374  _runtime‚13743çjß ü 0loss/FRt2/assignment_loss‚1.6158018944206498 1loss/FBt1/assignment_loss‚0.12446889735696086 /loss/FRt2/detection_loss‚1.3810025139307027 1loss/SRqqt2/detection_loss‚0.6911250638135803 2loss/SRqqt1/assignment_loss‚1.0850394552917746 2loss/SRqqt2/assignment_loss‚0.8285367623480292 -loss/FBt2/detection_loss‚1.46465163032214 _runtime‚ 1374.53045171 /loss/FRt1/assignment_loss‚2.991452740988402 0loss/FBt2/assignment_loss‚0.1289014810462959 .loss/FBt1/detection_loss‚1.716873674070806 &loss/total_loss‚14.521750845795225 % _timestamp‚1.7844617001423886e+09 _step‚163 /loss/FRt1/detection_loss‚0.5576753843169515 1loss/SRqqt1/detection_loss‚1.9362213478889316 epoch‚4 trainer/global_step‚19992ªNkà  _step‚163 /loss/FRt1/detection_loss‚0.5576753843169515 1loss/SRqqt1/detection_loss‚1.9362213478889316 epoch‚4 trainer/global_step‚1999 0loss/FRt2/assignment_loss‚1.6158018944206498 1loss/FBt1/assignment_loss‚0.12446889735696086 /loss/FRt2/detection_loss‚1.3810025139307027 1loss/SRqqt2/detection_loss‚0.6911250638135803 2loss/SRqqt1/assignment_loss‚1.0850394552917746 2loss/SRqqt2/assignment_loss‚0.8285367623480292 -loss/FBt2/detection_loss‚1.46465163032214 _runtime‚ 1374.53045171 /loss/FRt1/assignment_loss‚2.991452740988402 0loss/FBt2/assignment_loss‚0.1289014810462959 .loss/FBt1/detection_loss‚1.716873674070806 &loss/total_loss‚14.521750845795225 % _timestamp‚1.7844617001423886e+09£¡Ì$žá ‚(:“ ‰óòÒ©§È¤ gpu.0.gpu‚97.0 gpu.0.memory‚15, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚47.0 gpu.0.powerWatts‚207.668' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚69.22266666666667 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0éöŒïiâ ‚(:Þ ‰óòÒŽ—¥ proc.cpu.threads‚17 network.recv‚401194 disk.nvme1n1.in‚ 129.765625 disk.dm-0.in‚ 1982.2109375 disk.rbd0.out‚0" disk.nvme0c0n1.in‚ 1852.4453125' disk.nvme0c0n1.out‚20688.1220703125 disk.nvme0n1.in‚ 1852.4453125% disk.nvme0n1.out‚20688.1220703125$ disk.nvme1n1.out‚11909.705078125 disk.md0.in‚0 disk.dm-1.in‚0 disk.rbd0.in‚0 disk.nvme1c1n1.in‚ 129.765625$ memory_percent‚99.21343326568604 proc.memory.rssMB‚ 2827.5625* proc.memory.percent‚17.258071899414062 network.sent‚ 2.873e+06 disk.md0.out‚ 36.30078125# disk.dm-0.out‚31458.70556640625( disk./.usagePercent‚76.3286084310373" disk./.usageGB‚1171.8349609375" disk.dm-1.out‚1061.25732421875& disk.nvme1c1n1.out‚11909.705078125( proc.memory.availableMB‚ 128.87109375 cpu‚25.83417326993681G€2ã - _wandbruntime‚1388  _runtime‚1388Oƒxî=ä è{ :"Purity/1FRt*SRqqt*FBt/event_purity‚0.09898887574672699 :"Purity/1FRt2SRqqt1FBt/SRqqt_purity‚0.32499998807907104 /particle/accuracy_3_of_3‚0.9994933605194092 ("Purity/*FRt1SRqqt2FBt/event_purity‚0 7 Purity/1FRt*SRqqt1FBt/FBt_purity‚0.9980952143669128 8 Purity/1FRt0SRqqt*FBt/FRt_purity‚0.13621993362903595 :"Purity/2FRt1SRqqt0FBt/SRqqt_purity‚0.24749663472175598 ,&Purity/2FRt2SRqqt1FBt/event_proportion‚0 particle/accuracy_1_of_1‚1 % _timestamp‚1.7844617140169296e+09 & Purity/0FRt2SRqqt*FBt/FBt_purity‚1 :"Purity/*FRt1SRqqt2FBt/SRqqt_purity‚0.35185185074806213 ("Purity/2FRt1SRqqt0FBt/event_purity‚0 particle/accuracy_4_of_5‚1 8"Purity/*FRt1SRqqt1FBt/SRqqt_purity‚0.500253438949585 & Purity/*FRt2SRqqt1FBt/FBt_purity‚1 >&Purity/1FRt*SRqqt1FBt/event_proportion‚0.01643880270421505 7 Purity/1FRt*SRqqt2FBt/FBt_purity‚0.9753968119621277 :"Purity/2FRt*SRqqt1FBt/event_purity‚0.10500000417232513 @&Purity/2FRt0SRqqt1FBt/event_proportion‚0.0007672990905120969 -"Purity/2FRt*SRqqt2FBt/event_purity‚0.4375 jet/accuracy_4_of_5‚0 =&Purity/*FRt1SRqqt1FBt/event_proportion‚0.0188569575548172 +jet/accuracy_1_of_1‚0.16922669112682343 particle/accuracy_1_of_3‚1 _step‚164 7 Purity/0FRt0SRqqt1FBt/FBt_purity‚0.9940308928489685 =&Purity/1FRt*SRqqt0FBt/event_proportion‚0.6105840802192688 9"Purity/1FRt1SRqqt1FBt/SRqqt_purity‚0.5169419646263123 & Purity/2FRt1SRqqt2FBt/FBt_purity‚1 7 Purity/*FRt1SRqqt2FBt/FBt_purity‚0.7870370149612427 7 Purity/0FRt*SRqqt2FBt/FBt_purity‚0.9519038200378418 7 Purity/1FRt0SRqqt1FBt/FRt_purity‚0.5818801522254944 8&Purity/1FRt2SRqqt0FBt/event_proportion‚ 0.02294921875 /particle/accuracy_4_of_4‚0.9498758912086487 :"Purity/*FRt0SRqqt*FBt/event_purity‚0.19208215177059174 <&Purity/*FRt2SRqqt*FBt/event_proportion‚0.054222472012043 :"Purity/0FRt1SRqqt*FBt/event_purity‚0.25267040729522705 9"Purity/1FRt1SRqqt*FBt/SRqqt_purity‚0.2655978798866272 ("Purity/2FRt2SRqqt*FBt/event_purity‚0 =&Purity/0FRt0SRqqt*FBt/event_proportion‚0.0390159972012043 =&Purity/1FRt1SRqqt*FBt/event_proportion‚0.1534598171710968 ("Purity/1FRt1SRqqt2FBt/event_purity‚0 7 Purity/*FRt1SRqqt1FBt/FBt_purity‚0.9889274835586548 =&Purity/0FRt*SRqqt*FBt/event_proportion‚0.2839471697807312 7 Purity/0FRt*SRqqt1FBt/FBt_purity‚0.9908769726753235 ?&Purity/0FRt*SRqqt2FBt/event_proportion‚0.008068266324698925 =&Purity/1FRt*SRqqt*FBt/event_proportion‚0.6297432780265808 ("Purity/1FRt2SRqqt*FBt/event_purity‚0 7 Purity/0FRt0SRqqt2FBt/FBt_purity‚0.9619846343994141 9"Purity/1FRt1SRqqt2FBt/SRqqt_purity‚0.3333333432674408 & Purity/2FRt1SRqqt1FBt/FBt_purity‚1 ?&Purity/2FRt2SRqqt0FBt/event_proportion‚0.006417410913854837 & Purity/1FRt2SRqqt1FBt/FBt_purity‚1 9"Purity/2FRt*SRqqt*FBt/SRqqt_purity‚0.3030780553817749 =&Purity/2FRt*SRqqt0FBt/event_proportion‚0.0382486991584301 ("Purity/2FRt1SRqqt1FBt/event_purity‚0 ( Purity/2FRt1SRqqt2FBt/FRt_purity‚0.5 ) Purity/*FRt1SRqqt2FBt/FRt_purity‚0.75 9"Purity/0FRt*SRqqt1FBt/event_purity‚0.8157981038093567 9"Purity/*FRt2SRqqt0FBt/SRqqt_purity‚0.3235050439834595 9"Purity/0FRt2SRqqt0FBt/SRqqt_purity‚0.3026391565799713 ,&Purity/0FRt2SRqqt2FBt/event_proportion‚0 particle/accuracy_2_of_3‚1 +particle/sensitivity‚0.5640155673027039 ?&Purity/0FRt0SRqqt2FBt/event_proportion‚0.007626487873494625 6 Purity/0FRt1SRqqt*FBt/FBt_purity‚0.970343828201294 9"Purity/0FRt*SRqqt2FBt/SRqqt_purity‚0.3461538553237915 particle/accuracy_1_of_5‚1 7 Purity/*FRt0SRqqt2FBt/FBt_purity‚0.9713839292526245 =&Purity/0FRt1SRqqt0FBt/event_proportion‚0.2083565890789032 @&Purity/1FRt1SRqqt2FBt/event_proportion‚0.0002325148816453293 *jet/accuracy_1_of_2‚0.5426157712936401 epoch‚4 7 Purity/*FRt*SRqqt1FBt/FRt_purity‚0.5539023280143738 =&Purity/0FRt*SRqqt0FBt/event_proportion‚0.2322126179933548 =&Purity/1FRt0SRqqt*FBt/event_proportion‚0.4530552327632904 & Purity/2FRt0SRqqt2FBt/FBt_purity‚1 ("Purity/2FRt1SRqqt2FBt/SRqqt_purity‚0 particle/accuracy_2_of_5‚1 7 Purity/*FRt0SRqqt2FBt/FRt_purity‚0.8099018335342407 :"Purity/1FRt*SRqqt*FBt/SRqqt_purity‚0.28401708602905273 @&Purity/1FRt0SRqqt2FBt/event_proportion‚0.0024879092816263437 @&Purity/0FRt1SRqqt2FBt/event_proportion‚0.0004417782765813172 @&Purity/2FRt*SRqqt1FBt/event_proportion‚0.0016043527284637094 7 Purity/2FRt0SRqqt2FBt/FRt_purity‚0.7142857313156128 ("Purity/2FRt1SRqqt2FBt/event_purity‚0 jet/accuracy_5_of_5‚0 8"Purity/*FRt0SRqqt1FBt/event_purity‚0.859434962272644 :"Purity/*FRt2SRqqt0FBt/event_purity‚0.02818268910050392 =&Purity/0FRt0SRqqt0FBt/event_proportion‚0.0462472103536129 :"Purity/0FRt2SRqqt0FBt/event_purity‚0.06358211487531662 @&Purity/1FRt1SRqqt1FBt/event_proportion‚0.0064639137126505375 @&Purity/2FRt*SRqqt2FBt/event_proportion‚0.0002092633949359879 @&Purity/2FRt1SRqqt1FBt/event_proportion‚0.0008370535797439516 & Purity/2FRt0SRqqt1FBt/FBt_purity‚1 7 Purity/*FRt0SRqqt1FBt/FBt_purity‚0.9948660731315613 8"Purity/0FRt*SRqqt2FBt/event_purity‚0.869482159614563 9"Purity/0FRt0SRqqt2FBt/event_purity‚0.9239692687988281 @&Purity/*FRt1SRqqt2FBt/event_proportion‚0.0006975446594879031 7 Purity/1FRt*SRqqt1FBt/FRt_purity‚0.5565474033355713 ?&Purity/2FRt1SRqqt0FBt/event_proportion‚0.011928013525903225 jet/accuracy_1_of_5‚1 9"Purity/1FRt*SRqqt2FBt/SRqqt_purity‚0.3333333432674408 8 Purity/2FRt2SRqqt*FBt/FRt_purity‚0.20966096222400665 +jet/accuracy_2_of_4‚0.35112127661705017 9"Purity/*FRt*SRqqt2FBt/event_purity‚0.7682217359542847 8&Purity/*FRt2SRqqt0FBt/event_proportion‚ 0.05322265625 :"Purity/1FRt0SRqqt0FBt/event_purity‚0.12256282567977905 9"Purity/1FRt0SRqqt2FBt/event_purity‚0.5300642251968384 :"Purity/2FRt1SRqqt*FBt/SRqqt_purity‚0.27066197991371155 ?&Purity/2FRt2SRqqt*FBt/event_proportion‚0.006417410913854837 =&Purity/*FRt0SRqqt*FBt/event_proportion‚0.5129278302192688 :"Purity/*FRt0SRqqt0FBt/event_purity‚0.11742641776800156 :"Purity/*FRt1SRqqt*FBt/event_purity‚0.15033791959285736 7 Purity/0FRt0SRqqt*FBt/FBt_purity‚0.9844954609870911 *jet/accuracy_2_of_3‚0.2829212546348572 :"Purity/*FRt*SRqqt2FBt/SRqqt_purity‚0.35185185074806213 9"Purity/0FRt*SRqqt*FBt/SRqqt_purity‚0.2666644752025604 <"Purity/2FRt0SRqqt0FBt/event_purity‚0.0036324788816273212 8 Purity/2FRt1SRqqt*FBt/FRt_purity‚0.32697075605392456 :"Purity/*FRt*SRqqt*FBt/SRqqt_purity‚0.27518007159233093 7 Purity/*FRt1SRqqt*FBt/FRt_purity‚0.3337843418121338 8 Purity/*FRt2SRqqt1FBt/FRt_purity‚0.15000000596046448 & Purity/1FRt2SRqqt*FBt/FBt_purity‚1 +jet/accuracy_3_of_4‚0.06427829712629318 particle/accuracy_2_of_4‚1 7validation_average_jet_accuracy‚0.21246393024921417 8 Purity/*FRt0SRqqt0FBt/FRt_purity‚0.12230179458856583 @&Purity/0FRt2SRqqt1FBt/event_proportion‚0.0007207961170934141 A&Purity/1FRt2SRqqt1FBt/event_proportion‚0.00027901786961592734 & Purity/2FRt1SRqqt*FBt/FBt_purity‚1 particle/accuracy_1_of_4‚1 7 Purity/*FRt1SRqqt*FBt/FBt_purity‚0.9749889373779297 7 Purity/0FRt*SRqqt*FBt/FBt_purity‚0.9811074137687683 9"Purity/1FRt*SRqqt2FBt/event_purity‚0.4910052716732025 7 Purity/1FRt2SRqqt*FBt/FRt_purity‚0.2214592546224594 8 Purity/2FRt0SRqqt0FBt/FRt_purity‚0.12006676942110062 +particle/specificity‚0.8838698863983154 ;"Purity/1FRt1SRqqt0FBt/event_purity‚0.011202146299183369 7 Purity/2FRt*SRqqt1FBt/FRt_purity‚0.5300000309944153 9"Purity/2FRt0SRqqt*FBt/event_purity‚0.0181309562176466 8 Purity/2FRt2SRqqt0FBt/FRt_purity‚0.20966096222400665 9"Purity/2FRt2SRqqt0FBt/SRqqt_purity‚0.3433765172958374 *jet/accuracy_1_of_3‚0.7590119242668152 7 Purity/0FRt1SRqqt1FBt/FBt_purity‚0.9826170802116394 9"Purity/0FRt2SRqqt*FBt/event_purity‚0.0619041733443737 :"Purity/1FRt1SRqqt1FBt/event_purity‚0.12703721225261688 A&Purity/2FRt0SRqqt2FBt/event_proportion‚0.00018601190822664648 9"Purity/0FRt1SRqqt2FBt/SRqqt_purity‚0.3461538553237915 particle/accuracy_3_of_5‚1 ?&Purity/*FRt*SRqqt2FBt/event_proportion‚0.010997953824698925 =&Purity/1FRt0SRqqt0FBt/event_proportion‚0.4408714771270752 9"Purity/2FRt*SRqqt0FBt/SRqqt_purity‚0.2932918667793274 particle/accuracy_1_of_2‚1 7 Purity/*FRt0SRqqt*FBt/FBt_purity‚0.9877548217773438 =&Purity/*FRt0SRqqt1FBt/event_proportion‚0.0418526791036129 8 Purity/2FRt1SRqqt0FBt/FRt_purity‚0.31490975618362427 9"Purity/2FRt1SRqqt1FBt/SRqqt_purity‚0.6481481194496155 ,jet/accuracy_3_of_3‚0.040773723274469376 ("Purity/0FRt1SRqqt2FBt/event_purity‚0 *"Purity/2FRt0SRqqt2FBt/event_purity‚0.5 ;"Purity/*FRt2SRqqt*FBt/event_purity‚0.027685701847076416 8 Purity/*FRt1SRqqt0FBt/FRt_purity‚0.32377558946609497 9"Purity/0FRt*SRqqt1FBt/SRqqt_purity‚0.4688524901866913 =&Purity/0FRt1SRqqt*FBt/event_proportion‚0.2203543484210968 <"Purity/2FRt*SRqqt0FBt/event_purity‚0.0018638503970578313 @&Purity/2FRt1SRqqt2FBt/event_proportion‚2.325148852833081e-05 & Purity/2FRt0SRqqt*FBt/FBt_purity‚1 =&Purity/*FRt*SRqqt*FBt/event_proportion‚0.9537528157234192 7 Purity/*FRt0SRqqt*FBt/FRt_purity‚0.1366618424654007 :"Purity/1FRt1SRqqt0FBt/SRqqt_purity‚0.25439953804016113 7 Purity/1FRt1SRqqt1FBt/FRt_purity‚0.5322296619415283 8 Purity/1FRt2SRqqt0FBt/FRt_purity‚0.22263193130493164 <&Purity/2FRt*SRqqt*FBt/event_proportion‚0.040062315762043 8 Purity/2FRt0SRqqt*FBt/FRt_purity‚0.14137890934944153 =&Purity/2FRt1SRqqt*FBt/event_proportion‚0.0127883180975914 ("Purity/2FRt2SRqqt0FBt/event_purity‚0 7 Purity/0FRt1SRqqt2FBt/FBt_purity‚0.8269230723381042 7 Purity/*FRt*SRqqt1FBt/FBt_purity‚0.9931581616401672 :"Purity/1FRt0SRqqt*FBt/event_purity‚0.13221856951713562 ("Purity/1FRt2SRqqt1FBt/event_purity‚0 & Purity/2FRt*SRqqt*FBt/FBt_purity‚1 =&Purity/*FRt0SRqqt2FBt/event_proportion‚0.0103004090487957 =&Purity/*FRt1SRqqt*FBt/event_proportion‚0.3866024911403656 @&Purity/1FRt*SRqqt2FBt/event_proportion‚0.0027204242069274187 :"Purity/*FRt*SRqqt0FBt/SRqqt_purity‚0.26579639315605164 9"Purity/0FRt*SRqqt*FBt/event_purity‚0.3361886441707611 :"Purity/0FRt1SRqqt0FBt/event_purity‚0.24556702375411987 /"Purity/0FRt2SRqqt1FBt/SRqqt_purity‚0.328125 7 Purity/1FRt*SRqqt0FBt/FRt_purity‚0.1747560203075409 :"Purity/1FRt*SRqqt1FBt/event_purity‚0.32678166031837463 8 Purity/2FRt*SRqqt*FBt/FRt_purity‚0.21178072690963745 :"Purity/*FRt*SRqqt*FBt/event_purity‚0.16580531001091003 7 Purity/*FRt*SRqqt2FBt/FRt_purity‚0.8059335350990295 8"Purity/1FRt0SRqqt1FBt/event_purity‚0.467801034450531 =&Purity/1FRt1SRqqt0FBt/event_proportion‚0.1467633992433548 :"Purity/*FRt*SRqqt0FBt/event_purity‚0.12298822402954102 7 Purity/*FRt0SRqqt1FBt/FRt_purity‚0.5781079530715942 9"Purity/*FRt1SRqqt1FBt/event_purity‚0.2831871807575226 :"Purity/0FRt1SRqqt0FBt/SRqqt_purity‚0.24556702375411987 7 Purity/1FRt1SRqqt*FBt/FRt_purity‚0.3346743583679199 9"Purity/2FRt*SRqqt1FBt/SRqqt_purity‚0.6481481194496155 ("Purity/2FRt1SRqqt*FBt/event_purity‚0 7 Purity/*FRt*SRqqt2FBt/FBt_purity‚0.9587828516960144 :"Purity/*FRt1SRqqt*FBt/SRqqt_purity‚0.26144659519195557 >&Purity/1FRt0SRqqt1FBt/event_proportion‚0.00969587080180645 . Purity/2FRt0SRqqt1FBt/FRt_purity‚ 0.5546875 @&Purity/*FRt2SRqqt1FBt/event_proportion‚0.0009998140158131719 =&Purity/0FRt*SRqqt1FBt/event_proportion‚0.0436662957072258 7 Purity/1FRt1SRqqt*FBt/FBt_purity‚0.9835585355758667 +jet/accuracy_2_of_2‚0.08207961171865463 9"Purity/2FRt0SRqqt1FBt/event_purity‚0.2552083432674408 8 Purity/*FRt2SRqqt0FBt/FRt_purity‚0.21822449564933777 >&Purity/1FRt2SRqqt*FBt/event_proportion‚0.02322823740541935 =&Purity/*FRt1SRqqt0FBt/event_proportion‚0.3670479953289032 >&Purity/0FRt2SRqqt*FBt/event_proportion‚0.02457682229578495 7 Purity/1FRt*SRqqt*FBt/FBt_purity‚0.9918041229248047 (particle/accuracy‚0.8151777982711792 9"Purity/0FRt1SRqqt1FBt/event_purity‚0.3919731676578522 >&Purity/0FRt2SRqqt0FBt/event_proportion‚0.02385602705180645 :"Purity/1FRt*SRqqt0FBt/SRqqt_purity‚0.27599987387657166 7 Purity/1FRt0SRqqt2FBt/FRt_purity‚0.8130385875701904 8 Purity/1FRt*SRqqt*FBt/FRt_purity‚0.18752413988113403 7 Purity/1FRt0SRqqt1FBt/FBt_purity‚0.9960317015647888 :"Purity/1FRt2SRqqt0FBt/SRqqt_purity‚0.34496209025382996 8 Purity/2FRt*SRqqt0FBt/FRt_purity‚0.19580741226673126 *jet/accuracy_1_of_4‚0.8520953059196472 :"Purity/*FRt2SRqqt*FBt/SRqqt_purity‚0.32366707921028137 ,&Purity/2FRt2SRqqt2FBt/event_proportion‚0 :"Purity/*FRt*SRqqt1FBt/SRqqt_purity‚0.48311755061149597 ,&Purity/*FRt2SRqqt2FBt/event_proportion‚0 9"Purity/0FRt2SRqqt*FBt/SRqqt_purity‚0.3035910427570343 7 Purity/1FRt1SRqqt0FBt/FRt_purity‚0.3250576853752136 ("Purity/1FRt2SRqqt0FBt/event_purity‚0 =&Purity/2FRt0SRqqt*FBt/event_proportion‚0.0208565853536129 9"Purity/*FRt0SRqqt2FBt/event_purity‚0.8221557140350342 & Purity/*FRt2SRqqt*FBt/FBt_purity‚1 ("Purity/*FRt2SRqqt1FBt/event_purity‚0 8 Purity/1FRt2SRqqt1FBt/FRt_purity‚0.15000000596046448 7 Purity/2FRt1SRqqt1FBt/FRt_purity‚0.5138888955116272 & Purity/0FRt2SRqqt1FBt/FBt_purity‚1 ,&Purity/1FRt2SRqqt2FBt/event_proportion‚0 /particle/accuracy_3_of_4‚0.9970238208770752 particle/accuracy_5_of_5‚0 7 Purity/*FRt1SRqqt1FBt/FRt_purity‚0.5348438024520874 7 Purity/*FRt2SRqqt*FBt/FRt_purity‚0.2176928073167801 :"Purity/1FRt*SRqqt0FBt/event_purity‚0.09114839136600494 7 Purity/1FRt0SRqqt2FBt/FBt_purity‚0.9970238208770752 *loss/val_total_loss‚14.482613995278829 9"Purity/0FRt0SRqqt1FBt/event_purity‚0.9940308928489685 7 Purity/1FRt1SRqqt2FBt/FBt_purity‚0.7222222089767456 ;"Purity/2FRt*SRqqt*FBt/event_purity‚0.009360892698168755 + Purity/2FRt*SRqqt2FBt/FRt_purity‚0.6875 jet/accuracy_2_of_5‚1 8 Purity/*FRt*SRqqt*FBt/FRt_purity‚0.19021055102348328 9"Purity/0FRt0SRqqt*FBt/event_purity‚0.9815051555633545 9"Purity/0FRt*SRqqt0FBt/SRqqt_purity‚0.2560884952545166 :"Purity/1FRt2SRqqt*FBt/SRqqt_purity‚0.34506574273109436 ,jet/accuracy_4_of_4‚0.011098379269242287 7 Purity/*FRt*SRqqt*FBt/FBt_purity‚0.9846932291984558 :"Purity/*FRt1SRqqt0FBt/SRqqt_purity‚0.24888984858989716 =&Purity/0FRt0SRqqt1FBt/event_proportion‚0.0313895083963871 :"Purity/0FRt1SRqqt1FBt/SRqqt_purity‚0.48665350675582886 8 Purity/1FRt0SRqqt0FBt/FRt_purity‚0.12256282567977905 & Purity/2FRt*SRqqt1FBt/FBt_purity‚1 particle/accuracy_2_of_2‚1 <&Purity/*FRt*SRqqt0FBt/event_proportion‚0.881045401096344 9"Purity/0FRt*SRqqt0FBt/event_purity‚0.2268574982881546 :"Purity/0FRt1SRqqt*FBt/SRqqt_purity‚0.25855594873428345 :"Purity/1FRt1SRqqt*FBt/event_purity‚0.01609306037425995 9"Purity/2FRt2SRqqt*FBt/SRqqt_purity‚0.3433765172958374 =&Purity/*FRt*SRqqt1FBt/event_proportion‚0.0617094486951828 =&Purity/*FRt0SRqqt0FBt/event_proportion‚0.4607747495174408 9"Purity/*FRt1SRqqt0FBt/event_purity‚0.1438409984111786 & Purity/1FRt1SRqqt1FBt/FBt_purity‚1 9"Purity/*FRt*SRqqt1FBt/event_purity‚0.6678659319877625 7 Purity/1FRt1SRqqt2FBt/FRt_purity‚0.7777777910232544 & Purity/2FRt*SRqqt2FBt/FBt_purity‚1 :"Purity/*FRt2SRqqt1FBt/SRqqt_purity‚0.32175925374031067 ("Purity/0FRt0SRqqt0FBt/event_purity‚1 &particle/f_score‚0.567214846611023 jet/accuracy_3_of_5‚1 _runtime‚1388.405876141 ?&Purity/0FRt1SRqqt1FBt/event_proportion‚0.011555989272892475 ("Purity/0FRt2SRqqt1FBt/event_purity‚0 9"Purity/1FRt*SRqqt1FBt/SRqqt_purity‚0.5068466067314148 7 Purity/1FRt*SRqqt2FBt/FRt_purity‚0.8129252195358276 ("Purity/2FRt*SRqqt2FBt/SRqqt_purity‚0 trainer/global_step‚1999 8 Purity/*FRt*SRqqt0FBt/FRt_purity‚0.17701292037963867 6 Purity/1FRt0SRqqt*FBt/FBt_purity‚0.996457576751709 >&Purity/2FRt0SRqqt0FBt/event_proportion‚0.01990327425301075È«€8-å í{ & Purity/2FRt*SRqqt1FBt/FBt_purity‚1 particle/accuracy_2_of_2‚1 <&Purity/*FRt*SRqqt0FBt/event_proportion‚0.881045401096344 9"Purity/0FRt*SRqqt0FBt/event_purity‚0.2268574982881546 :"Purity/0FRt1SRqqt*FBt/SRqqt_purity‚0.25855594873428345 :"Purity/1FRt1SRqqt*FBt/event_purity‚0.01609306037425995 9"Purity/2FRt2SRqqt*FBt/SRqqt_purity‚0.3433765172958374 =&Purity/*FRt*SRqqt1FBt/event_proportion‚0.0617094486951828 =&Purity/*FRt0SRqqt0FBt/event_proportion‚0.4607747495174408 9"Purity/*FRt1SRqqt0FBt/event_purity‚0.1438409984111786 & Purity/1FRt1SRqqt1FBt/FBt_purity‚1 9"Purity/*FRt*SRqqt1FBt/event_purity‚0.6678659319877625 7 Purity/1FRt1SRqqt2FBt/FRt_purity‚0.7777777910232544 & Purity/2FRt*SRqqt2FBt/FBt_purity‚1 :"Purity/*FRt2SRqqt1FBt/SRqqt_purity‚0.32175925374031067 ("Purity/0FRt0SRqqt0FBt/event_purity‚1 &particle/f_score‚0.567214846611023 jet/accuracy_3_of_5‚1 _runtime‚1388.405876141 ?&Purity/0FRt1SRqqt1FBt/event_proportion‚0.011555989272892475 ("Purity/0FRt2SRqqt1FBt/event_purity‚0 9"Purity/1FRt*SRqqt1FBt/SRqqt_purity‚0.5068466067314148 7 Purity/1FRt*SRqqt2FBt/FRt_purity‚0.8129252195358276 ("Purity/2FRt*SRqqt2FBt/SRqqt_purity‚0 trainer/global_step‚1999 8 Purity/*FRt*SRqqt0FBt/FRt_purity‚0.17701292037963867 6 Purity/1FRt0SRqqt*FBt/FBt_purity‚0.996457576751709 >&Purity/2FRt0SRqqt0FBt/event_proportion‚0.01990327425301075 :"Purity/1FRt*SRqqt*FBt/event_purity‚0.09898887574672699 :"Purity/1FRt2SRqqt1FBt/SRqqt_purity‚0.32499998807907104 /particle/accuracy_3_of_3‚0.9994933605194092 ("Purity/*FRt1SRqqt2FBt/event_purity‚0 7 Purity/1FRt*SRqqt1FBt/FBt_purity‚0.9980952143669128 8 Purity/1FRt0SRqqt*FBt/FRt_purity‚0.13621993362903595 :"Purity/2FRt1SRqqt0FBt/SRqqt_purity‚0.24749663472175598 ,&Purity/2FRt2SRqqt1FBt/event_proportion‚0 particle/accuracy_1_of_1‚1 % _timestamp‚1.7844617140169296e+09 & Purity/0FRt2SRqqt*FBt/FBt_purity‚1 :"Purity/*FRt1SRqqt2FBt/SRqqt_purity‚0.35185185074806213 ("Purity/2FRt1SRqqt0FBt/event_purity‚0 particle/accuracy_4_of_5‚1 8"Purity/*FRt1SRqqt1FBt/SRqqt_purity‚0.500253438949585 & Purity/*FRt2SRqqt1FBt/FBt_purity‚1 >&Purity/1FRt*SRqqt1FBt/event_proportion‚0.01643880270421505 7 Purity/1FRt*SRqqt2FBt/FBt_purity‚0.9753968119621277 :"Purity/2FRt*SRqqt1FBt/event_purity‚0.10500000417232513 @&Purity/2FRt0SRqqt1FBt/event_proportion‚0.0007672990905120969 -"Purity/2FRt*SRqqt2FBt/event_purity‚0.4375 jet/accuracy_4_of_5‚0 =&Purity/*FRt1SRqqt1FBt/event_proportion‚0.0188569575548172 +jet/accuracy_1_of_1‚0.16922669112682343 particle/accuracy_1_of_3‚1 _step‚164 7 Purity/0FRt0SRqqt1FBt/FBt_purity‚0.9940308928489685 =&Purity/1FRt*SRqqt0FBt/event_proportion‚0.6105840802192688 9"Purity/1FRt1SRqqt1FBt/SRqqt_purity‚0.5169419646263123 & Purity/2FRt1SRqqt2FBt/FBt_purity‚1 7 Purity/*FRt1SRqqt2FBt/FBt_purity‚0.7870370149612427 7 Purity/0FRt*SRqqt2FBt/FBt_purity‚0.9519038200378418 7 Purity/1FRt0SRqqt1FBt/FRt_purity‚0.5818801522254944 8&Purity/1FRt2SRqqt0FBt/event_proportion‚ 0.02294921875 /particle/accuracy_4_of_4‚0.9498758912086487 :"Purity/*FRt0SRqqt*FBt/event_purity‚0.19208215177059174 <&Purity/*FRt2SRqqt*FBt/event_proportion‚0.054222472012043 :"Purity/0FRt1SRqqt*FBt/event_purity‚0.25267040729522705 9"Purity/1FRt1SRqqt*FBt/SRqqt_purity‚0.2655978798866272 ("Purity/2FRt2SRqqt*FBt/event_purity‚0 =&Purity/0FRt0SRqqt*FBt/event_proportion‚0.0390159972012043 =&Purity/1FRt1SRqqt*FBt/event_proportion‚0.1534598171710968 ("Purity/1FRt1SRqqt2FBt/event_purity‚0 7 Purity/*FRt1SRqqt1FBt/FBt_purity‚0.9889274835586548 =&Purity/0FRt*SRqqt*FBt/event_proportion‚0.2839471697807312 7 Purity/0FRt*SRqqt1FBt/FBt_purity‚0.9908769726753235 ?&Purity/0FRt*SRqqt2FBt/event_proportion‚0.008068266324698925 =&Purity/1FRt*SRqqt*FBt/event_proportion‚0.6297432780265808 ("Purity/1FRt2SRqqt*FBt/event_purity‚0 7 Purity/0FRt0SRqqt2FBt/FBt_purity‚0.9619846343994141 9"Purity/1FRt1SRqqt2FBt/SRqqt_purity‚0.3333333432674408 & Purity/2FRt1SRqqt1FBt/FBt_purity‚1 ?&Purity/2FRt2SRqqt0FBt/event_proportion‚0.006417410913854837 & Purity/1FRt2SRqqt1FBt/FBt_purity‚1 9"Purity/2FRt*SRqqt*FBt/SRqqt_purity‚0.3030780553817749 =&Purity/2FRt*SRqqt0FBt/event_proportion‚0.0382486991584301 ("Purity/2FRt1SRqqt1FBt/event_purity‚0 ( Purity/2FRt1SRqqt2FBt/FRt_purity‚0.5 ) Purity/*FRt1SRqqt2FBt/FRt_purity‚0.75 9"Purity/0FRt*SRqqt1FBt/event_purity‚0.8157981038093567 9"Purity/*FRt2SRqqt0FBt/SRqqt_purity‚0.3235050439834595 9"Purity/0FRt2SRqqt0FBt/SRqqt_purity‚0.3026391565799713 ,&Purity/0FRt2SRqqt2FBt/event_proportion‚0 particle/accuracy_2_of_3‚1 +particle/sensitivity‚0.5640155673027039 ?&Purity/0FRt0SRqqt2FBt/event_proportion‚0.007626487873494625 6 Purity/0FRt1SRqqt*FBt/FBt_purity‚0.970343828201294 9"Purity/0FRt*SRqqt2FBt/SRqqt_purity‚0.3461538553237915 particle/accuracy_1_of_5‚1 7 Purity/*FRt0SRqqt2FBt/FBt_purity‚0.9713839292526245 =&Purity/0FRt1SRqqt0FBt/event_proportion‚0.2083565890789032 @&Purity/1FRt1SRqqt2FBt/event_proportion‚0.0002325148816453293 *jet/accuracy_1_of_2‚0.5426157712936401 epoch‚4 7 Purity/*FRt*SRqqt1FBt/FRt_purity‚0.5539023280143738 =&Purity/0FRt*SRqqt0FBt/event_proportion‚0.2322126179933548 =&Purity/1FRt0SRqqt*FBt/event_proportion‚0.4530552327632904 & Purity/2FRt0SRqqt2FBt/FBt_purity‚1 ("Purity/2FRt1SRqqt2FBt/SRqqt_purity‚0 particle/accuracy_2_of_5‚1 7 Purity/*FRt0SRqqt2FBt/FRt_purity‚0.8099018335342407 :"Purity/1FRt*SRqqt*FBt/SRqqt_purity‚0.28401708602905273 @&Purity/1FRt0SRqqt2FBt/event_proportion‚0.0024879092816263437 @&Purity/0FRt1SRqqt2FBt/event_proportion‚0.0004417782765813172 @&Purity/2FRt*SRqqt1FBt/event_proportion‚0.0016043527284637094 7 Purity/2FRt0SRqqt2FBt/FRt_purity‚0.7142857313156128 ("Purity/2FRt1SRqqt2FBt/event_purity‚0 jet/accuracy_5_of_5‚0 8"Purity/*FRt0SRqqt1FBt/event_purity‚0.859434962272644 :"Purity/*FRt2SRqqt0FBt/event_purity‚0.02818268910050392 =&Purity/0FRt0SRqqt0FBt/event_proportion‚0.0462472103536129 :"Purity/0FRt2SRqqt0FBt/event_purity‚0.06358211487531662 @&Purity/1FRt1SRqqt1FBt/event_proportion‚0.0064639137126505375 @&Purity/2FRt*SRqqt2FBt/event_proportion‚0.0002092633949359879 @&Purity/2FRt1SRqqt1FBt/event_proportion‚0.0008370535797439516 & Purity/2FRt0SRqqt1FBt/FBt_purity‚1 7 Purity/*FRt0SRqqt1FBt/FBt_purity‚0.9948660731315613 8"Purity/0FRt*SRqqt2FBt/event_purity‚0.869482159614563 9"Purity/0FRt0SRqqt2FBt/event_purity‚0.9239692687988281 @&Purity/*FRt1SRqqt2FBt/event_proportion‚0.0006975446594879031 7 Purity/1FRt*SRqqt1FBt/FRt_purity‚0.5565474033355713 ?&Purity/2FRt1SRqqt0FBt/event_proportion‚0.011928013525903225 jet/accuracy_1_of_5‚1 9"Purity/1FRt*SRqqt2FBt/SRqqt_purity‚0.3333333432674408 8 Purity/2FRt2SRqqt*FBt/FRt_purity‚0.20966096222400665 +jet/accuracy_2_of_4‚0.35112127661705017 9"Purity/*FRt*SRqqt2FBt/event_purity‚0.7682217359542847 8&Purity/*FRt2SRqqt0FBt/event_proportion‚ 0.05322265625 :"Purity/1FRt0SRqqt0FBt/event_purity‚0.12256282567977905 9"Purity/1FRt0SRqqt2FBt/event_purity‚0.5300642251968384 :"Purity/2FRt1SRqqt*FBt/SRqqt_purity‚0.27066197991371155 ?&Purity/2FRt2SRqqt*FBt/event_proportion‚0.006417410913854837 =&Purity/*FRt0SRqqt*FBt/event_proportion‚0.5129278302192688 :"Purity/*FRt0SRqqt0FBt/event_purity‚0.11742641776800156 :"Purity/*FRt1SRqqt*FBt/event_purity‚0.15033791959285736 7 Purity/0FRt0SRqqt*FBt/FBt_purity‚0.9844954609870911 *jet/accuracy_2_of_3‚0.2829212546348572 :"Purity/*FRt*SRqqt2FBt/SRqqt_purity‚0.35185185074806213 9"Purity/0FRt*SRqqt*FBt/SRqqt_purity‚0.2666644752025604 <"Purity/2FRt0SRqqt0FBt/event_purity‚0.0036324788816273212 8 Purity/2FRt1SRqqt*FBt/FRt_purity‚0.32697075605392456 :"Purity/*FRt*SRqqt*FBt/SRqqt_purity‚0.27518007159233093 7 Purity/*FRt1SRqqt*FBt/FRt_purity‚0.3337843418121338 8 Purity/*FRt2SRqqt1FBt/FRt_purity‚0.15000000596046448 & Purity/1FRt2SRqqt*FBt/FBt_purity‚1 +jet/accuracy_3_of_4‚0.06427829712629318 particle/accuracy_2_of_4‚1 7validation_average_jet_accuracy‚0.21246393024921417 8 Purity/*FRt0SRqqt0FBt/FRt_purity‚0.12230179458856583 @&Purity/0FRt2SRqqt1FBt/event_proportion‚0.0007207961170934141 A&Purity/1FRt2SRqqt1FBt/event_proportion‚0.00027901786961592734 & Purity/2FRt1SRqqt*FBt/FBt_purity‚1 particle/accuracy_1_of_4‚1 7 Purity/*FRt1SRqqt*FBt/FBt_purity‚0.9749889373779297 7 Purity/0FRt*SRqqt*FBt/FBt_purity‚0.9811074137687683 9"Purity/1FRt*SRqqt2FBt/event_purity‚0.4910052716732025 7 Purity/1FRt2SRqqt*FBt/FRt_purity‚0.2214592546224594 8 Purity/2FRt0SRqqt0FBt/FRt_purity‚0.12006676942110062 +particle/specificity‚0.8838698863983154 ;"Purity/1FRt1SRqqt0FBt/event_purity‚0.011202146299183369 7 Purity/2FRt*SRqqt1FBt/FRt_purity‚0.5300000309944153 9"Purity/2FRt0SRqqt*FBt/event_purity‚0.0181309562176466 8 Purity/2FRt2SRqqt0FBt/FRt_purity‚0.20966096222400665 9"Purity/2FRt2SRqqt0FBt/SRqqt_purity‚0.3433765172958374 *jet/accuracy_1_of_3‚0.7590119242668152 7 Purity/0FRt1SRqqt1FBt/FBt_purity‚0.9826170802116394 9"Purity/0FRt2SRqqt*FBt/event_purity‚0.0619041733443737 :"Purity/1FRt1SRqqt1FBt/event_purity‚0.12703721225261688 A&Purity/2FRt0SRqqt2FBt/event_proportion‚0.00018601190822664648 9"Purity/0FRt1SRqqt2FBt/SRqqt_purity‚0.3461538553237915 particle/accuracy_3_of_5‚1 ?&Purity/*FRt*SRqqt2FBt/event_proportion‚0.010997953824698925 =&Purity/1FRt0SRqqt0FBt/event_proportion‚0.4408714771270752 9"Purity/2FRt*SRqqt0FBt/SRqqt_purity‚0.2932918667793274 particle/accuracy_1_of_2‚1 7 Purity/*FRt0SRqqt*FBt/FBt_purity‚0.9877548217773438 =&Purity/*FRt0SRqqt1FBt/event_proportion‚0.0418526791036129 8 Purity/2FRt1SRqqt0FBt/FRt_purity‚0.31490975618362427 9"Purity/2FRt1SRqqt1FBt/SRqqt_purity‚0.6481481194496155 ,jet/accuracy_3_of_3‚0.040773723274469376 ("Purity/0FRt1SRqqt2FBt/event_purity‚0 *"Purity/2FRt0SRqqt2FBt/event_purity‚0.5 ;"Purity/*FRt2SRqqt*FBt/event_purity‚0.027685701847076416 8 Purity/*FRt1SRqqt0FBt/FRt_purity‚0.32377558946609497 9"Purity/0FRt*SRqqt1FBt/SRqqt_purity‚0.4688524901866913 =&Purity/0FRt1SRqqt*FBt/event_proportion‚0.2203543484210968 <"Purity/2FRt*SRqqt0FBt/event_purity‚0.0018638503970578313 @&Purity/2FRt1SRqqt2FBt/event_proportion‚2.325148852833081e-05 & Purity/2FRt0SRqqt*FBt/FBt_purity‚1 =&Purity/*FRt*SRqqt*FBt/event_proportion‚0.9537528157234192 7 Purity/*FRt0SRqqt*FBt/FRt_purity‚0.1366618424654007 :"Purity/1FRt1SRqqt0FBt/SRqqt_purity‚0.25439953804016113 7 Purity/1FRt1SRqqt1FBt/FRt_purity‚0.5322296619415283 8 Purity/1FRt2SRqqt0FBt/FRt_purity‚0.22263193130493164 <&Purity/2FRt*SRqqt*FBt/event_proportion‚0.040062315762043 8 Purity/2FRt0SRqqt*FBt/FRt_purity‚0.14137890934944153 =&Purity/2FRt1SRqqt*FBt/event_proportion‚0.0127883180975914 ("Purity/2FRt2SRqqt0FBt/event_purity‚0 7 Purity/0FRt1SRqqt2FBt/FBt_purity‚0.8269230723381042 7 Purity/*FRt*SRqqt1FBt/FBt_purity‚0.9931581616401672 :"Purity/1FRt0SRqqt*FBt/event_purity‚0.13221856951713562 ("Purity/1FRt2SRqqt1FBt/event_purity‚0 & Purity/2FRt*SRqqt*FBt/FBt_purity‚1 =&Purity/*FRt0SRqqt2FBt/event_proportion‚0.0103004090487957 =&Purity/*FRt1SRqqt*FBt/event_proportion‚0.3866024911403656 @&Purity/1FRt*SRqqt2FBt/event_proportion‚0.0027204242069274187 :"Purity/*FRt*SRqqt0FBt/SRqqt_purity‚0.26579639315605164 9"Purity/0FRt*SRqqt*FBt/event_purity‚ïÂ¥»0.3361886441707611 :"Purity/0FRt1SRqqt0FBt/event_purity‚0.24556702375411987 /"Purity/0FRt2SRqqt1FBt/SRqqt_purity‚0.328125 7 Purity/1FRt*SRqqt0FBt/FRt_purity‚0.1747560203075409 :"Purity/1FRt*SRqqt1FBt/event_purity‚0.32678166031837463 8 Purity/2FRt*SRqqt*FBt/FRt_purity‚0.21178072690963745 :"Purity/*FRt*SRqqt*FBt/event_purity‚0.16580531001091003 7 Purity/*FRt*SRqqt2FBt/FRt_purity‚0.8059335350990295 8"Purity/1FRt0SRqqt1FBt/event_purity‚0.467801034450531 =&Purity/1FRt1SRqqt0FBt/event_proportion‚0.1467633992433548 :"Purity/*FRt*SRqqt0FBt/event_purity‚0.12298822402954102 7 Purity/*FRt0SRqqt1FBt/FRt_purity‚0.5781079530715942 9"Purity/*FRt1SRqqt1FBt/event_purity‚0.2831871807575226 :"Purity/0FRt1SRqqt0FBt/SRqqt_purity‚0.24556702375411987 7 Purity/1FRt1SRqqt*FBt/FRt_purity‚0.3346743583679199 9"Purity/2FRt*SRqqt1FBt/SRqqt_purity‚0.6481481194496155 ("Purity/2FRt1SRqqt*FBt/event_purity‚0 7 Purity/*FRt*SRqqt2FBt/FBt_purity‚0.9587828516960144 :"Purity/*FRt1SRqqt*FBt/SRqqt_purity‚0.26144659519195557 >&Purity/1FRt0SRqqt1FBt/event_proportion‚0.00969587080180645 . Purity/2FRt0SRqqt1FBt/FRt_purity‚ 0.5546875 @&Purity/*FRt2SRqqt1FBt/event_proportion‚0.0009998140158131719 =&Purity/0FRt*SRqqt1FBt/event_proportion‚0.0436662957072258 7 Purity/1FRt1SRqqt*FBt/FBt_purity‚0.9835585355758667 +jet/accuracy_2_of_2‚0.08207961171865463 9"Purity/2FRt0SRqqt1FBt/event_purity‚0.2552083432674408 8 Purity/*FRt2SRqqt0FBt/FRt_purity‚0.21822449564933777 >&Purity/1FRt2SRqqt*FBt/event_proportion‚0.02322823740541935 =&Purity/*FRt1SRqqt0FBt/event_proportion‚0.3670479953289032 >&Purity/0FRt2SRqqt*FBt/event_proportion‚0.02457682229578495 7 Purity/1FRt*SRqqt*FBt/FBt_purity‚0.9918041229248047 (particle/accuracy‚0.8151777982711792 9"Purity/0FRt1SRqqt1FBt/event_purity‚0.3919731676578522 >&Purity/0FRt2SRqqt0FBt/event_proportion‚0.02385602705180645 :"Purity/1FRt*SRqqt0FBt/SRqqt_purity‚0.27599987387657166 7 Purity/1FRt0SRqqt2FBt/FRt_purity‚0.8130385875701904 8 Purity/1FRt*SRqqt*FBt/FRt_purity‚0.18752413988113403 7 Purity/1FRt0SRqqt1FBt/FBt_purity‚0.9960317015647888 :"Purity/1FRt2SRqqt0FBt/SRqqt_purity‚0.34496209025382996 8 Purity/2FRt*SRqqt0FBt/FRt_purity‚0.19580741226673126 *jet/accuracy_1_of_4‚0.8520953059196472 :"Purity/*FRt2SRqqt*FBt/SRqqt_purity‚0.32366707921028137 ,&Purity/2FRt2SRqqt2FBt/event_proportion‚0 :"Purity/*FRt*SRqqt1FBt/SRqqt_purity‚0.48311755061149597 ,&Purity/*FRt2SRqqt2FBt/event_proportion‚0 9"Purity/0FRt2SRqqt*FBt/SRqqt_purity‚0.3035910427570343 7 Purity/1FRt1SRqqt0FBt/FRt_purity‚0.3250576853752136 ("Purity/1FRt2SRqqt0FBt/event_purity‚0 =&Purity/2FRt0SRqqt*FBt/event_proportion‚0.0208565853536129 9"Purity/*FRt0SRqqt2FBt/event_purity‚0.8221557140350342 & Purity/*FRt2SRqqt*FBt/FBt_purity‚1 ("Purity/*FRt2SRqqt1FBt/event_purity‚0 8 Purity/1FRt2SRqqt1FBt/FRt_purity‚0.15000000596046448 7 Purity/2FRt1SRqqt1FBt/FRt_purity‚0.5138888955116272 & Purity/0FRt2SRqqt1FBt/FBt_purity‚1 ,&Purity/1FRt2SRqqt2FBt/event_proportion‚0 /particle/accuracy_3_of_4‚0.9970238208770752 particle/accuracy_5_of_5‚0 7 Purity/*FRt1SRqqt1FBt/FRt_purity‚0.5348438024520874 7 Purity/*FRt2SRqqt*FBt/FRt_purity‚0.2176928073167801 :"Purity/1FRt*SRqqt0FBt/event_purity‚0.09114839136600494 7 Purity/1FRt0SRqqt2FBt/FBt_purity‚0.9970238208770752 *loss/val_total_loss‚14.482613995278829 9"Purity/0FRt0SRqqt1FBt/event_purity‚0.9940308928489685 7 Purity/1FRt1SRqqt2FBt/FBt_purity‚0.7222222089767456 ;"Purity/2FRt*SRqqt*FBt/event_purity‚0.009360892698168755 + Purity/2FRt*SRqqt2FBt/FRt_purity‚0.6875 jet/accuracy_2_of_5‚1 8 Purity/*FRt*SRqqt*FBt/FRt_purity‚0.19021055102348328 9"Purity/0FRt0SRqqt*FBt/event_purity‚0.9815051555633545 9"Purity/0FRt*SRqqt0FBt/SRqqt_purity‚0.2560884952545166 :"Purity/1FRt2SRqqt*FBt/SRqqt_purity‚0.34506574273109436 ,jet/accuracy_4_of_4‚0.011098379269242287 7 Purity/*FRt*SRqqt*FBt/FBt_purity‚0.9846932291984558 :"Purity/*FRt1SRqqt0FBt/SRqqt_purity‚0.24888984858989716 =&Purity/0FRt0SRqqt1FBt/event_proportion‚0.0313895083963871 :"Purity/0FRt1SRqqt1FBt/SRqqt_purity‚0.48665350675582886 8 Purity/1FRt0SRqqt0FBt/FRt_purity‚0.12256282567977905¤ß4ÿqæ ‚(:æ ˜óòÒŸ¸ç¤) disk./.usagePercent‚76.32859973444688 disk.nvme0n1.in‚ 1852.4453125 disk.md0.in‚0" disk.nvme0c0n1.in‚ 1852.4453125( proc.memory.availableMB‚ 191.68359375 proc.memory.rssMB‚ 2827.5625 cpu‚25.850151411125108 network.sent‚ 2.935623e+06% disk./.usageGB‚1171.8348274230957$ disk.nvme1n1.out‚11974.658203125 disk.dm-0.in‚ 1982.2109375 disk.dm-1.in‚0" disk.dm-1.out‚1125.17138671875 disk.rbd0.out‚0( disk.nvme0c0n1.out‚20690.96142578125& disk.nvme0n1.out‚20690.96142578125 disk.nvme1c1n1.in‚ 129.765625 disk.md0.out‚ 36.69140625 disk.rbd0.in‚0 proc.cpu.threads‚17& disk.nvme1c1n1.out‚11974.658203125 disk.nvme1n1.in‚ 129.765625! disk.dm-0.out‚31461.724609375$ memory_percent‚98.83005619049072* proc.memory.percent‚17.258071899414062 network.recv‚405663â}mç ‚(:’ ˜óòÒçš™¥ gpu.0.gpu‚54.0 gpu.0.memory‚9, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚48.0 gpu.0.powerWatts‚220.888' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚73.62933333333334 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚09=-qè ‚(:æ §óòÒ¸‡ð¤ network.recv‚409053 disk.dm-1.in‚0 disk.nvme0n1.in‚ 1852.4453125 disk.md0.in‚0# disk.dm-0.out‚31463.35107421875' proc.memory.availableMB‚ 191.4296875 proc.memory.rssMB‚ 2827.5625 cpu‚25.84716157819339 network.sent‚ 2.979789e+06) disk./.usagePercent‚76.32856668740331 disk.dm-1.out‚1189.103515625 disk.rbd0.in‚0 disk.rbd0.out‚0 disk.nvme1n1.in‚ 129.765625 disk.md0.out‚ 37.05078125" disk.nvme0c0n1.in‚ 1852.4453125( disk.nvme0c0n1.out‚20692.77783203125& disk.nvme0n1.out‚20692.77783203125 disk.nvme1c1n1.in‚ 129.765625% disk.nvme1n1.out‚12039.1708984375 disk.dm-0.in‚ 1982.2109375$ memory_percent‚98.83160591125488 proc.cpu.threads‚17% disk./.usageGB‚1171.8343200683594' disk.nvme1c1n1.out‚12039.1708984375* proc.memory.percent‚17.258071899414062 ½¾žé ‚(:“ §óòÒ´üò¤ gpu.0.gpu‚87.0 gpu.0.memory‚21, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚47.0 gpu.0.powerWatts‚113.635' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚37.87833333333334 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ŽÊf2ê - _wandbruntime‚1420  _runtime‚1420]Çë Á ' lr-AdamW/pg1‚0.00013682484154696687 ' lr-AdamW/pg2‚0.00013682484154696687 trainer/global_step‚2049 $ _timestamp‚1.784461745747512e+09 _runtime‚ 1420.13588685 _step‚165O³qÌì Æ $ _timestamp‚1.784461745747512e+09 _runtime‚ 1420.13588685 _step‚165 ' lr-AdamW/pg1‚0.00013682484154696687 ' lr-AdamW/pg2‚0.00013682484154696687 trainer/global_step‚2049¥u |2í - _wandbruntime‚1420  _runtime‚1420£¦Ýà'î ÚO PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚16252421 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚10907376 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚157181349894832 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚160749577678336 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚20616598 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _runtime‚1420.136624332 _step‚166 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚4650408 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚159058438640128 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1691244737536 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚4650228 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 % _timestamp‚1.7844617457483978e+09 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1691156077568 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚184 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚160749577678336 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1691156077568 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚10907372 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚120886150602240 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚119144426515968 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚160749683377664 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1741836752896 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚16252423 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚119144520625664 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 112666624 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1686518222902 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚36869019 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚158867851078374 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1691244737536 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚20626741 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 % _timestamp‚1.7844617457483978e+09 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1691156077568 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚184 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚160749577678336 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1691156077568 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚10907372 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚120886150602240 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚119144426515968 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚160749683377664 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1741836752896 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚16252423 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚119144520625664 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 112666624 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1686518222902 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚36869019 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚158867851078374 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1691244737536 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚20626741 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 IDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚10912432 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚120943096359936 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚20636575 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚159133571267072 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚160825513060352 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚4652566 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1742532115968 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚15564998 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 _runtime‚1420.542026694 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1691941793280 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚119200658353664 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚16260050 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚119200564243968 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.cur·˜™¿ rent‚ 121048064 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚20626418 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚120943311517696 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚159133588306432 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚158942925098366 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1687388504040 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚36896627 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚160825618766848 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚20626418 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚4652390 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 % _timestamp‚1.7844617461534674e+09 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚15564818 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 _step‚167 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚16260050 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚36886468 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚159133571267072 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚10912428 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚10912428 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚10912432 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚120943096359936 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚20636575 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚159133571267072 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚160825513060352 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚4652566 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1742532115968 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚15564998 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 _runtime‚1420.542026694 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1691941793280 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚119200658353664 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚16260050 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚119200564243968 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 121048064 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚20626418 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚120943311517696 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚159133588306432 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚158942925098366 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1687388504040 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚36896627 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚160825618766848 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚20626418 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚4652390 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 % _timestamp‚1.7844617461534674e+09 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚15564818 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 _step‚167§þTÐ2ó - _wandbruntime‚1420  _runtime‚1420ðtô  .loss/FRt2/detection_loss‚1.426356860891575 1loss/SRqqt2/detection_loss‚0.7923445861674225 /loss/FBt2/detection_loss‚1.7303742575548067 epoch‚5 2loss/SRqqt2/assignment_loss‚0.7892202261784983 /loss/FRt1/detection_loss‚0.6002607334363681 &loss/total_loss‚14.274806224455318 trainer/global_step‚2049 _step‚168 0loss/FRt1/assignment_loss‚2.9455782608734653 2loss/SRqqt1/assignment_loss‚1.2160071750557182 1loss/FBt2/assignment_loss‚0.30040819537233193 1loss/SRqqt1/detection_loss‚1.7014448732288197 /loss/FBt1/detection_loss‚1.1745329749264128 % _timestamp‚1.7844617461544409e+09 _runtime‚1420.542575434 0loss/FRt2/assignment_loss‚1.5119535442767085 1loss/FBt1/assignment_loss‚0.08632453649319029eXë{ õ † .loss/FRt2/detection_loss‚1.426356860891575 1loss/SRqqt2/detection_loss‚0.7923445861674225 /loss/FBt2/detection_loss‚1.7303742575548067 epoch‚5 2loss/SRqqt2/assignment_loss‚0.7892202261784983 /loss/FRt1/detection_loss‚0.6002607334363681 &loss/total_loss‚14.274806224455318 trainer/global_step‚2049 _step‚168 0loss/FRt1/assignment_loss‚2.9455782608734653 2loss/SRqqt1/assignment_loss‚1.2160071750557182 1loss/FBt2/assignment_loss‚0.30040819537233193 1loss/SRqqt1/detection_loss‚1.7014448732288197 /loss/FBt1/detection_loss‚1.1745329749264128 % _timestamp‚1.7844617461544409e+09 _runtime‚1420.542575434 0loss/FRt2/assignment_loss‚1.5119535442767085 1loss/FBt1/assignment_loss‚0.08632453649319029¨ÁïWiö ‚(:Þ ¶óòÒ«ò»¥( disk./.usagePercent‚76.3285642026632" disk.dm-0.out‚31465.6533203125 disk.nvme1c1n1.in‚ 129.765625' disk.nvme1c1n1.out‚12039.8251953125$ memory_percent‚98.82893562316895 disk.dm-0.in‚ 1982.2109375 disk.rbd0.out‚0" disk.nvme0c0n1.in‚ 1852.4453125# disk.nvme0n1.out‚20695.19921875 disk.md0.in‚0' proc.memory.availableMB‚ 191.8671875 proc.cpu.threads‚17% disk./.usageGB‚1171.8342819213867 disk.dm-1.in‚0 disk.dm-1.out‚1189.146484375% disk.nvme0c0n1.out‚20695.19921875 disk.md0.out‚ 37.39453125 proc.memory.rssMB‚ 2827.5625 cpu‚25.84473691382853 disk.rbd0.in‚0 disk.nvme1n1.in‚ 129.765625% disk.nvme1n1.out‚12039.8251953125* proc.memory.percent‚17.258071899414062 disk.nvme0n1.in‚ 1852.4453125 network.sent‚ 2.983693e+06 network.recv‚411753w5Dž÷ ‚(:“ ¶óò҈젥 gpu.0.gpu‚91.0 gpu.0.memory‚20, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚49.0 gpu.0.powerWatts‚226.733' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚75.57766666666666 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0[¦Û2jø ‚(:ß ÅóòÒ٭٤ disk.nvme0n1.in‚ 1852.4453125 disk.md0.out‚ 37.6328125* proc.memory.percent‚17.258071899414062 proc.cpu.threads‚17 network.recv‚415543) disk./.usagePercent‚76.32856072402703" disk.nvme0c0n1.in‚ 1852.4453125& disk.nvme0c0n1.out‚20696.048828125$ disk.nvme0n1.out‚20696.048828125 disk.nvme1c1n1.in‚ 129.765625' disk.nvme1c1n1.out‚12040.3291015625% disk.nvme1n1.out‚12040.3291015625 network.sent‚ 3.046055e+06 disk.dm-1.in‚0 disk.rbd0.in‚0 disk.rbd0.out‚0 disk.md0.in‚0$ memory_percent‚98.82757663726807( proc.memory.availableMB‚ 192.08984375 disk.dm-1.out‚ 1189.2890625 disk.nvme1n1.in‚ 129.765625 proc.memory.rssMB‚ 2827.5625 cpu‚25.842754778789875$ disk./.usageGB‚1171.834228515625 disk.dm-0.in‚ 1982.2109375" disk.dm-0.out‚31466.3291015625>z?“ù ‚(:ˆ Åóòғ𣥠gpu.0.gpu‚43.0 gpu.0.memory‚10, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚47.0 gpu.0.powerWatts‚122.817' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚40.939 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0̦ü2ú - _wandbruntime‚1452  _runtime‚1452z„ïLÈû  ' lr-AdamW/pg1‚0.00013667692038545694 ' lr-AdamW/pg2‚0.00013667692038545694 trainer/global_step‚2099 $ _timestamp‚1.784461778366902e+09 _runtime‚1452.755396476 _step‚169ä§îÍü Ç ' lr-AdamW/pg1‚0.00013667692038545694 ' lr-AdamW/pg2‚0.00013667692038545694 trainer/global_step‚2099 $ _timestamp‚1.784461778366902e+09 _runtime‚1452.755396476 _step‚169©7ÐÌæ2ý - _wandbruntime‚1452  _runtime‚1452 lÖmà'þ ÚO G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚123733612781056 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚121951312915968 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚164546516633088 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1730594677760 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚160895859125072 _step‚170 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1730506017792 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚16633873 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚21109450 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚15919172 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1782299865088 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1730506017792 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚16633871 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚164546516633088 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚123733832140288 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚16633873 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚37743321 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚179 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 219359232 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚11160172 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372523008 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚162815921955328 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚183 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚121951407025664 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 125249536 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚162621719704568 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚164546410933760 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚162815904915968 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚11160176 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 trainer/global_step‚2099 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚4758996 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚4758817 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1782425114624 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1725843540136 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚164546410933760 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚162621615878782 % _timestamp‚1.7844617783677843e+09 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1730594677760 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚21119593 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚164546410933760 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚162621615878782 % _timestamp‚1.7844617783677843e+09 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1730594677760 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚21119593 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1730506017792 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚16633871 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚164546516633088 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚123733832140288 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚16633873 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚37743321 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚179 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 219359232 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚11160172 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372523008 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚162815921955328 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚183 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚121951407025664 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 125249536 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚162621719704568 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚164546410933760 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚162815904915968 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚11160176 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 trainer/global_step‚2099 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚4758996 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚4758817 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1782425114624 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1725843540136 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚164622346315776 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚21119270 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1726540248310 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚37770929 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚162891071621632 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1731291733504 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚37770929 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚15926204 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚4760976 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚123790783272960 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚11165232 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1731380400640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 % _timestamp‚1.7844617787817695e+09 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚122007544753664 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1783238519296 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 129436672 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚160970166689824 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0û›_¢î&‚ èM NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚16641502 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚15926386 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372523008 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚122007450643968 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚162891054582272 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚178 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚162696689898774 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚164622346315776 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚21119270 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1726540248310 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚37770929 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚162891071621632 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1731291733504 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚37770929 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚15926204 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚4760976 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚123790783272960 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚11165232 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1731380400640 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 % _timestamp‚1.7844617787817695e+09 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚122007544753664 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1783238519296 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 129436672 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚160970166689824 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚164622452022272 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 _step‚171 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚11165228 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚162696793724672 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 FDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚168343244189184 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚166573405270528 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚17015323 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚11412972 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚17015321 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚17015321 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚166573388231168 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚166573405270528 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚16279272 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 % _timestamp‚1.7844618116450992e+09 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚166375380679190 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚21612445 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1769855958016 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 _runtime‚1486.033504172 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚168343349888512 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚166573388231168 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚168343349888512 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 123152384 _step‚174 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚126581090525184 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1823014361600 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚21612445 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚176 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚124758293425664 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚164610385394672 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚4866296 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1765082070944 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1769944617984 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 217262080 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚172 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1764995284518 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚166375484504976 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1769944617984 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1769855958016 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚126581307787264 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚164610402434032 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚17015323 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚21602302hNˆå'‘ ßO MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚126581090525184 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1823014361600 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚21612445 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚176 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚124758293425664 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚164610385394672 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚4866296 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1765082070944 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1769944617984 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 217262080 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚172 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1764995284518 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚166375484504976 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1769944617984 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1769855958016 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚126581307787264 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚164610402434032 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚17015323 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚21602302 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚4866124 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚38627768 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚16279096 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1822891209216 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚168343244189184 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚166573405270528 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚17015323 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚11412972 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚17015321 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚17015321 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚166573388231168 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚166573405270528 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚16279272 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 % _timestamp‚1.7844618116450992e+09 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚166375380679190 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚21612445 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1769855958016 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 _runtime‚1486.033504172 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚168343349888512 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚166573388231168 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚168343349888512 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 123152384 _step‚174®dSª2’ - _wandbruntime‚1486  _runtime‚1486ˆÝ"Kê&“ äM H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚164684692959424 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚16286272 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372523008 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚38645231 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1823705923584 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚168419285277696 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚21622279 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚126638249736704 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1765778779118 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 _runtime‚1486.448543745 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚21622279 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚181 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 112659456 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚164684675920064 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚126638042967552 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚38635072 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1770730340864 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 206769152 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚166648554936832 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1765865565656 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1770641673728 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚38645231 % _timestamp‚1.7844618120599635e+09 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚168419179571200 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 _step‚175 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚17022950 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 trainer/global_step‚2149 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚21612122 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1770730340864 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚124814337043968 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚168419285277696 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚166648537897472 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚166648554936832 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚166450558525080 d½Ç¸” éM XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1770641673728 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚38635072 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚11418028 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚4868244 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚17022952 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚185 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚4868425 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1823818583040 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 BÍ7!_end/oversize_segments.peak‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚166450454699182 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚16286272 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372523008 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚38645231 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1823705923584 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚168419285277696 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚21622279 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚126638249736704 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1765778779118 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 _runtime‚1486.448543745 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚21622279 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚181 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 112659456 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚164684675920064 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚126638042967552 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚38635072 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1770730340864 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 206769152 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚166648554936832 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1765865565656 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1770641673728 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚38645231 % _timestamp‚1.7844618120599635e+09 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚168419179571200 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 _step‚175 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚17022950 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 trainer/global_step‚2149 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚21612122 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1770730340864 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚124814337043968 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚168419285277696 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚166648537897472 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚166648554936832 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚166450558525080 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚164684692959424 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 EDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚170330871546368 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1809205898240 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1804233815326 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚17396771 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚16639024 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚11665772 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1863483173376 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 123152384 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚129428445736960 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚11665776 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚170330888585728 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚22095154 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚168324911664272 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚192 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚17396771 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚127565085715968 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1863360020992 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚172140077444608 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚22105297 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚22095154 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚172140183143936 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 _step‚178 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372523008 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚17396773 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 trainer/global_step‚2199 % _timestamp‚1.7844618455535076e+09 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚17396773 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1809294558208 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1809294558208 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚170129249305384 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚39491925 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚22105297 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚170129145479598 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚172140183143936 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚4973440 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚172140077444608 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚170330888585728 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚170330871546368 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚16639216 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 217262080 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚168324928703632 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚127565179825664 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚39502070 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚129428662999040 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 _step‚178 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372523008 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚17396773 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 trainer/global_step‚2199 % _timestamp‚1.7844618455535076e+09 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚17396773 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1809294558208 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1809294558208 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚170129249305384 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚39491925 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚22105297 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚170129145479598 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚172140183143936 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚4973440 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚172140077444608 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚170330888585728 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚170330871546368 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚16639216 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 217262080 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚168324928703632 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚127565179825664 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚39502070 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚129428662999040 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚170330871546368 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1809205898240 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1804233815326 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚17396771 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚16639024 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚11665772 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1863483173376 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 123152384 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚129428445736960 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚11665776 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_"†±pool.peak‚ 267417279 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚170330888585728 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚22095154 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚168324911664272 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚192 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0²4ç̘2¢ - _wandbruntime‚1520  _runtime‚1520É‘ø-ê&£ äM J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚17404400 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚170406038252032 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚16646206 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚39509374 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚11670832 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚17404402 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 _runtime‚1520.363444465 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚172216118533120 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1810080281088 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚129485389422080 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚170204219499590 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚22104974 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1810080281088 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚129485615065600 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 225643520 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚22104974 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚168399219229024 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚168399202189664 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚172216012826624 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚4975378 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚4975561 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚127621317553664 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1864165978112 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚39509374 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚172216118533120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1809991613952 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚39519533 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚17404400 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚22115131 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372523008 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚11670828 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 % _timestamp‚1.7844618459748445e+09 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚16646393 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚22115131 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚187 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1864297511936 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 trainer/global_step‚2199 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1809991613952 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1805104096464 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚170204323325488 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚16646206 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚39509374 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚11670832 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚17404402 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 _runtime‚1520.363444465 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚172216118533120 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1810080281088 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚129485389422080 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚170204219499590 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚22104974 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1810080281088 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚129485615065600 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 225643520 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚22104974 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚168399219229024 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚168399202189664 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚172216012826624 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚4975378 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚4975561 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚127621317553664 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1864165978112 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚39509374 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚172216118533120 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1809991613952 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚39519533 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚17404400 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚22115131 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372523008 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚11670828 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 % _timestamp‚1.7844618459748445e+09 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚16646393 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚22115131 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚187 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1864297511936 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 trainer/global_step‚2199 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1809991613952 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1805104096464 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚170204323325488 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812³Ï&¦‚2¥ - _wandbruntime‚1520  _runtime‚1520(§ä¦ ý 2loss/SRqqt2/assignment_loss‚0.8161470686460234 1loss/FBt1/assignment_loss‚0.10090047876405048 1loss/SRqqt1/detection_loss‚1.9079458723528742 1loss/SRqqt2/detection_loss‚0.7319064993917709 trainer/global_step‚2199 % _timestamp‚1.7844618459766502e+09 _step‚180 epoch‚5 0loss/FBt2/assignment_loss‚0.3234128786970014 .loss/FRt1/detection_loss‚0.543564858972417 /loss/FRt2/detection_loss‚1.3629294811464716 .loss/FBt2/detection_loss‚1.551464072846133 &loss/total_loss‚13.935480350994638 _runtime‚1520.364876627 /loss/FRt1/assignment_loss‚3.175612147027403 0loss/FRt2/assignment_loss‚1.4140786218249108 .loss/FBt1/detection_loss‚1.084097633896476 2loss/SRqqt1/assignment_loss‚0.9234207374291064ý<©2§ ‚ /loss/FRt1/assignment_loss‚3.175612147027403 0loss/FRt2/assignment_loss‚1.4140786218249108 .loss/FBt1/detection_loss‚1.084097633896476 2loss/SRqqt1/assignment_loss‚0.9234207374291064 2loss/SRqqt2/assignment_loss‚0.8161470686460234 1loss/FBt1/assignment_loss‚0.10090047876405048 1loss/SRqqt1/detection_loss‚1.9079458723528742 1loss/SRqqt2/detection_loss‚0.7319064993917709 trainer/global_step‚2199 % _timestamp‚1.7844618459766502e+09 _step‚180 epoch‚5 0loss/FBt2/assignment_loss‚0.3234128786970014 .loss/FRt1/detection_loss‚0.543564858972417 /loss/FRt2/detection_loss‚1.3629294811464716 .loss/FBt2/detection_loss‚1.551464072846133 &loss/total_loss‚13.935480350994638 _runtime‚1520.364876627´µ›6`r¨ ‚(:ç ŸôòÒ¢¯Ê¤ network.sent‚ 3.190213e+06" disk.nvme0c0n1.in‚ 1852.4453125$ memory_percent‚98.82931709289551' proc.memory.availableMB‚ 191.8046875# proc.memory.rssMB‚ 2827.56640625 proc.cpu.threads‚17% disk./.usageGB‚1171.5971336364746 disk.dm-1.in‚0" disk.dm-1.out‚1193.16064453125 disk.rbd0.out‚0 disk.nvme1c1n1.in‚ 129.765625& disk.nvme1n1.out‚12047.97119140625) disk./.usagePercent‚76.31311731877379 disk.rbd0.in‚0% disk.nvme0c0n1.out‚20706.81640625 disk.nvme0n1.in‚ 1852.4453125# disk.nvme0n1.out‚20706.81640625( disk.nvme1c1n1.out‚12047.97119140625 disk.nvme1n1.in‚ 129.765625 disk.dm-0.in‚ 1982.2109375 network.recv‚437342 disk.md0.in‚0 disk.md0.out‚ 39.66015625! disk.dm-0.out‚31476.498046875* proc.memory.percent‚17.258095741271973 cpu‚25.849660289774047¤Söî“© ‚(:ˆ ŸôòÒÀÔ°¥ gpu.0.gpu‚96.0 gpu.0.memory‚23, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚47.0 gpu.0.powerWatts‚113.949' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚37.983 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0óTÞ9zª ‚(:ï ®ôòÒ™ôÈ¥ network.sent‚ 3.254621e+06 disk.dm-0.in‚ 1982.2109375( disk.nvme1c1n1.out‚12050.07275390625& disk.nvme1n1.out‚12050.07275390625) disk./.usagePercent‚76.31309644695679% disk./.usageGB‚1171.5968132019043 disk.md0.in‚0 disk.dm-1.in‚0 disk.rbd0.out‚0( disk.nvme0c0n1.out‚20713.44775390625 disk.nvme1n1.in‚ 129.765625$ memory_percent‚98.82867336273193 network.recv‚441777# disk.dm-0.out‚31484.30126953125 disk.rbd0.in‚0 disk.nvme0n1.in‚ 1852.4453125& disk.nvme0n1.out‚20713.44775390625 disk.nvme1c1n1.in‚ 129.765625( proc.memory.availableMB‚ 191.91015625 cpu‚25.84674166133724 disk.md0.out‚ 40.08203125" disk.dm-1.out‚1193.19970703125" disk.nvme0c0n1.in‚ 1852.4453125# proc.memory.rssMB‚ 2827.56640625* proc.memory.percent‚17.258095741271973 proc.cpu.threads‚17@Ù$žž« ‚(:“ ®ôòÒ’üû¥ gpu.0.gpu‚90.0 gpu.0.memory‚17, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚49.0 gpu.0.powerWatts‚221.321' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚73.77366666666667 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0[?Ã2¬ - _wandbruntime‚1554  _runtime‚1554݉ÀÆ­ À & lr-AdamW/pg2‚0.0001362073551412344 trainer/global_step‚2249 $ _timestamp‚1.784461879730535e+09 _runtime‚1554.119805307 _step‚181 & lr-AdamW/pg1‚0.00013620735514123446¾cË® Å _runtime‚1554.119805307 _step‚181 & lr-AdamW/pg1‚0.0001362073551412344 & lr-AdamW/pg2‚0.0001362073551412344 trainer/global_step‚2249 $ _timestamp‚1.784461879730535e+09µi,¦‰2¯ - _wandbruntime‚1554  _runtime‚1554zÛ…° ÚO TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚11918572 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚130371972115968 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 _step‚182 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚173882910280006 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚172039437933872 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1848555838464 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚22588006 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 trainer/global_step‚2249 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚22598149 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚175936910700032 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚175936910700032 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 =6DeviceStatsMonitor.on_train_batch_start/max_split_sïe^[ ize‚-1 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚174088371900928 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1848644498432 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚17778221 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚174088354861568 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1843559132560 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1848644498432 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚173883014105792 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1848555838464 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚11918576 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372523008 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚130372066225664 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1903989086720 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚16998927 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚22588006 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚172039454973232 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚16998744 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚17778221 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚132275833856000 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚132276055312384 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960U:å'± ßO G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1848555838464 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚22588006 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 trainer/global_step‚2249 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚22598149 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚175936910700032 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚175936910700032 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚174088371900928 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1848644498432 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚17778221 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚174088354861568 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1843559132560 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1848644498432 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚173883014105792 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1848555838464 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚11918576 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372523008 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚130372066225664 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 1903989086720 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚16998927 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚22588006 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚172039454973232 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚16998744 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚17778221 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚132275833856000 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚132276055312384 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚11918572 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚130371972115968 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 _step‚182 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚173882910280006 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚172039437933872¶–Uo2² - _wandbruntime‚1554  _runtime‚1554F}5 æ&³ àM HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 $ _timestamp‚1.784461880154066e+09 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1849341554176 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚17005839 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1849341554176 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚17785850 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚176012951788544 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1849430221312 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚174163504527872 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚130428203953664 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚173958088125896 _step‚183 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚178 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚22607983 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚132332774764032 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚40393835 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚130428109843968 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚132333002504704 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚174163521567232 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚172113745498624 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚176012951788544 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚174163521567232 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚5082385 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 227740672 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚40383676 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚17785850 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚22607983 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1844342627272 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚22597826 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚17785852 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚173957984299998 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚17006017 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚172113728459264 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1904798551040 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚174163504527872 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 trainer/global_step‚2249 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1849430221312 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 133630976 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚40393835 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4F_'l…$´ åM _runtime‚ 1554.542716 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚176012846082048 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚22597826 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚17785850 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚176012951788544 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1849430221312 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚174163504527872 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚130428203953664 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚173958088125896 _step‚183 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚178 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚22607983 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚132332774764032 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚40393835 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚130428109843968 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚132333002504704 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚174163521567232 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚172113745498624 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚176012951788544 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚174163521567232 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚5082385 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 227740672 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚40383676 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚17785850 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚22607983 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1844342627272 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚22597826 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚17785852 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚173957984299998 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚17006017 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚172113728459264 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1904798551040 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚174163504527872 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 trainer/global_step‚2249 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1849430221312 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 133630976 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚40393835 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 $ _timestamp‚1.784461880154066e+09 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1849341554176 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚17005839 @‘ý½ˆf:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1849341554176 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1944362590208 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚177636778906200 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1887905778688 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚18159671 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚177845855216128 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 $ _timestamp‚1.784461913163502e+09 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚179733849654784 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1887994438656 _step‚186 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚179733743955456 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚41250674 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 219359232 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1882710876942 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚135123440465408 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚177636675080414 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 _runtime‚1587.551861327 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1887905778688 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 trainer/global_step‚2299 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5186430 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372523008 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚23091001 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚133178952625664 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚133178858515968 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚177845855216128 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696Ó¬ä'Á ÞO R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚179733743955456 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚41250674 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 219359232 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 1882710876942 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚135123440465408 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚177636675080414 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 _runtime‚1587.551861327 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1887905778688 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 trainer/global_step‚2299 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5186430 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372523008 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚23091001 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚133178952625664 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚133178858515968 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚177845855216128 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚175753981242832 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1887994438656 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚177845838176768 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚177845838176768 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚163 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 125249536 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 1882797663368 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚18159671 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1944362590208 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚177636778906200 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 1887905778688 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚18159671 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚177845855216128 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 $ _timestamp‚1.784461913163502e+09 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚179733849654784 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1887994438656 _step‚186º ñW92 - _wandbruntime‚1587  _runtime‚1587#Ê Êôà äM JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚41268137 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚23090678 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚172 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚41257978 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚12176432 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚176 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚5188803 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1888780161536 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚179809785043968 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚17365059 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚177920987843072 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚179809679337472 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 _step‚187 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚18167302 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1888691494400 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚133235090353664 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚23100835 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 _runtime‚1587.967543791 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚179809785043968 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚17365235 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚12176428 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1883581158080 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5188631 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚135180162513920 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1945166269952 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1883494371542 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚41257978 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚177921004882432 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚23090678 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚135180383963136 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372523008 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C=DeviceStatsM¿aQ&ö onitor.on_train_batch_end/oversize_segments.freed‚0 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚18167300 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 trainer/global_step‚2299 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚41268137 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚175828271768224 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚17365059 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚177920987843072 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚179809679337472 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 _step‚187 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚18167302 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 1888691494400 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚133235090353664 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚23100835 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 _runtime‚1587.967543791 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚179809785043968 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚17365235 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚12176428 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1883581158080 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5188631 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚135180162513920 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1945166269952 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1883494371542 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚41257978 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚177921004882432 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚23090678 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚135180383963136 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372523008 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚18167300 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 trainer/global_step‚2299 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚41268137 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚175828271768224 FDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 116860928 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚183530577210880 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚18541123 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚166 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚137970606577152 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚181603338531328 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚18541121 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚181603321491968 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚42114831 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1984861661184 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1927344378880 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 116860928 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚183530577210880 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚18541123 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚166 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚137970606577152 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚181603338531328 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚18541121 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 TDDeviceStatsMonitor.on_train_batch_start/reserv2åned_bytes.all.allocated‚ 14638120960 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚181603321491968 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚42114831 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 1984861661184 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1927344378880 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 IDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚42142439 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚181678471158272 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚181678471158272 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚179542798037824 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5295886 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1985789166080 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚17725114 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚157 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 _step‚191 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1922819688888 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚12429228 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚181678488197632 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 213060608 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚136041882643968 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1985670215168 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚23593687 _runtime‚1621.508568213 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚42132280 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚23583530 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚42142439 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1928130101760 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚183606618299392 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚161 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 % _timestamp‚1.7844619471200671e+09 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚183606618299392 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚181465617726712 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚18548750 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚18548750 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚1307gJï&Ö éM RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚23593687 _runtime‚1621.508568213 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚42132280 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚23583530 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚42142439 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1928130101760 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚183606618299392 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚161 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 % _timestamp‚1.7844619471200671e+09 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚183606618299392 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚181465617726712 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚18548750 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚18548750 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1928130101760 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚42142439 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚181678471158272 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚181678471158272 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚179542798037824 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5295886 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 1985789166080 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚17725114 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚157 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 _step‚191 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1922819688888 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚12429228 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚181678488197632 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 213060608 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚136041882643968 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 1985670215168 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200¿ä2× - _wandbruntime‚1621  _runtime‚1621‘©8¼Ø þ 0loss/FRt2/assignment_loss‚1.1872591031850248 &loss/total_loss‚12.889549342816167 trainer/global_step‚2349 _step‚192 epoch‚5 1loss/SRqqt2/assignment_loss‚0.792613158647684 1loss/FBt1/assignment_loss‚0.13375831258986906 /loss/FRt1/detection_loss‚0.6268458426495794 /loss/FRt2/detection_loss‚1.4425540777963604 1loss/SRqqt1/detection_loss‚1.5816447551174022 .loss/FBt2/detection_loss‚1.331135527999623 % _timestamp‚1.7844619471210723e+09 /loss/FRt1/assignment_loss‚3.036987233024904 1loss/FBt2/assignment_loss‚0.21334150327636267 /loss/FBt1/detection_loss‚0.7994203199408693 _runtime‚1621.509281594 1loss/SRqqt1/assignment_loss‚1.043319441773808 1loss/SRqqt2/detection_loss‚0.7006700668146809«x$* Ù ƒ epoch‚5 1loss/SRqqt2/assignment_loss‚0.792613158647684 1loss/FBt1/assignment_loss‚0.13375831258986906 /loss/FRt1/detection_loss‚0.6268458426495794 /loss/FRt2/detection_loss‚1.4425540777963604 1loss/SRqqt1/detection_loss‚1.5816447551174022 .loss/FBt2/detection_loss‚1.331135527999623 % _timestamp‚1.7844619471210723e+09 /loss/FRt1/assignment_loss‚3.036987233024904 1loss/FBt2/assignment_loss‚0.21334150327636267 /loss/FBt1/detection_loss‚0.7994203199408693 _runtime‚1621.509281594 1loss/SRqqt1/assignment_loss‚1.043319441773808 1loss/SRqqt2/detection_loss‚0.7006700668146809 0loss/FRt2/assignment_loss‚1.1872591031850248 &loss/total_loss‚12.889549342816167 trainer/global_step‚2349 _step‚192À+<Ä«rÚ ‚(:ç ˆõòÒ²ýí¤( disk./.usagePercent‚76.1361533821726& disk.nvme0n1.out‚20954.66845703125' disk.nvme1c1n1.out‚12083.6865234375 disk.md0.in‚0 disk.dm-1.in‚0" disk.dm-1.out‚1193.85302734375 disk.rbd0.in‚0# proc.memory.rssMB‚ 2827.56640625 network.sent‚ 3.40971e+06 network.recv‚462824% disk./.usageGB‚1168.8802947998047 disk.dm-0.in‚ 1982.2109375 disk.rbd0.out‚0 proc.cpu.threads‚17 disk.md0.out‚ 42.203125" disk.nvme0c0n1.in‚ 1852.4453125( disk.nvme0c0n1.out‚20954.66845703125 disk.nvme0n1.in‚ 1852.4453125% disk.nvme1n1.out‚12083.6865234375# memory_percent‚98.8297700881958* proc.memory.percent‚17.258095741271973 cpu‚25.83147389183822 disk.nvme1c1n1.in‚ 129.765625 disk.nvme1n1.in‚ 129.765625" disk.dm-0.out‚31753.8974609375( proc.memory.availableMB‚ 191.73046875DeviceStaãU¼Æ"#tsMonitor.on_train_batch_start/active.large_pool.peak‚1509 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚187327410466304 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚18077268 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚5400464 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 223553536 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1966694319104 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚12676972 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚24066562 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚168 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚42989133 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1966605659136 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚172 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚12676976 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚185360821846528 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚18922573 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚18077440 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚185144308507016 % _timestamp‚1.7844619797664108e+09dF¤¥å'ã ßO F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚24066562 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚168 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚42989133 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 1966605659136 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚172 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚12676976 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚185360821846528 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚18922573 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚18077440 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚185144308507016 % _timestamp‚1.7844619797664108e+09 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 1966694319104 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚187327516165632 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2025365808640 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚140818220678144 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2025495252480 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚183183016742672 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚18922571 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚187327410466304 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚18077268 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚5400464 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 223553536 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 1966694319104 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚12676972 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812Â-^ŠV2ä - _wandbruntime‚1654  _runtime‚16545蟾ê&å äM C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚185219382527120 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚187403451554816 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚185435954473472 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚18930200 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚138848863153664 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚187403345848320 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚43006582 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1961971433158 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 133630976 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚183257324307424 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚24086539 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚183257307268064 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚185435971512832 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1967480041984 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚24076382 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5402395 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚185219278701222 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1962058219696 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1967391374848 trainer/global_step‚2399 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚24076382 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚43006582 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚18930202 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚187403451554816 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚18930200 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1967480041984 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 _runtime‚1654.575357925 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 227740672 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚140874943144448 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372523008 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚43016741 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚140875170885120 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚43016741 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚24086539 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚185435954473472 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚12682032 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚162 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚18084423 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚162 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚18084423 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚187403345848320 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚43006582 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.aa)öÔ5ll.current‚ 103825898 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 1961971433158 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 133630976 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚183257324307424 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚24086539 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚183257307268064 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚185435971512832 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 1967480041984 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚24076382 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5402395 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚185219278701222 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 1962058219696 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 1967391374848 trainer/global_step‚2399 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚24076382 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚43006582 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚18930202 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚187403451554816 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚18930200 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 1967480041984 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 _runtime‚1654.575357925 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 227740672 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚140874943144448 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372523008 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚43016741 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚140875170885120 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682ÃY)2ç - _wandbruntime‚1654  _runtime‚1654L±upè þ 2loss/SRqqt1/assignment_loss‚0.7600184678627374 /loss/FBt1/detection_loss‚0.9939689189144165 %loss/total_loss‚12.99368066743713 trainer/global_step‚2399 2loss/SRqqt2/assignment_loss‚0.8506385636889006 /loss/FRt1/detection_loss‚0.5996848552884515 $ _timestamp‚1.784461980188631e+09 0loss/FBt2/assignment_loss‚0.1731117334117395 /loss/FRt2/detection_loss‚1.3358394663603619 1loss/SRqqt1/detection_loss‚1.5232284653880825 1loss/SRqqt2/detection_loss‚0.6977460149619504 /loss/FBt2/detection_loss‚1.6087433542707004 /loss/FRt1/assignment_loss‚2.899235965263459 0loss/FRt2/assignment_loss‚1.4137894917797569 1loss/FBt1/assignment_loss‚0.13767537024657409 epoch‚5 _runtime‚1654.576887718 _step‚196ø’^ é ƒ 2loss/SRqqt1/assignment_loss‚0.7600184678627374 /loss/FBt1/detection_loss‚0.9939689189144165 %loss/total_loss‚12.99368066743713 trainer/global_step‚2399 2loss/SRqqt2/assignment_loss‚0.8506385636889006 /loss/FRt1/detection_loss‚0.5996848552884515 $ _timestamp‚1.784461980188631e+09 0loss/FBt2/assignment_loss‚0.1731117334117395 /loss/FRt2/detection_loss‚1.3358394663603619 1loss/SRqqt1/detection_loss‚1.5232284653880825 1loss/SRqqt2/detection_loss‚0.6977460149619504 /loss/FBt2/detection_loss‚1.6087433542707004 /loss/FRt1/assignment_loss‚2.899235965263459 0loss/FRt2/assignment_loss‚1.4137894917797569 1loss/FBt1/assignment_loss‚0.13767537024657409 epoch‚5 _runtime‚1654.576887718 _step‚196ÄúNrê ‚(:ç ¦õòÒ‚ disk.rbd0.out‚0" disk.nvme0c0n1.in‚ 1852.4453125 disk.nvme0n1.in‚ 1852.4453125$ disk.nvme0n1.out‚21018.431640625 network.sent‚ 3.476835e+06) disk./.usagePercent‚76.15581860935974 disk.md0.in‚0" disk.dm-0.out‚31818.0380859375" disk.dm-1.out‚1194.09716796875 disk.nvme1n1.in‚ 129.765625& disk.nvme1n1.out‚12085.66259765625$ memory_percent‚99.22571182250977% disk./.usageGB‚1169.1822052001953 disk.rbd0.in‚0& disk.nvme0c0n1.out‚21018.431640625 disk.nvme1c1n1.in‚ 129.765625 cpu‚25.850237276221176 proc.cpu.threads‚17 network.recv‚470470 disk.md0.out‚ 42.828125 disk.dm-0.in‚ 1982.2109375( disk.nvme1c1n1.out‚12085.66259765625& proc.memory.availableMB‚ 126.859375# proc.memory.rssMB‚ 2827.56640625* proc.memory.percent‚17.258095741271973 disk.dm-1.in‚0;ƒä^žë ‚(:“ ¦õòÒúôª¥ gpu.0.gpu‚82.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚46.0 gpu.0.powerWatts‚112.939' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚37.64633333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0˜¾o2ì - _wandbruntime‚1669  _runtime‚1669éÏ&½ÿ=í ù{ ,&Purity/1FRt2SRqqt2FBt/event_proportion‚0 9"Purity/2FRt1SRqqt1FBt/SRqqt_purity‚0.6481481194496155 8"Purity/*FRt0SRqqt1FBt/event_purity‚0.871714174747467 7 Purity/*FRt*SRqqt0FBt/FRt_purity‚0.1779700070619583 8 Purity/*FRt0SRqqt*FBt/FRt_purity‚0.16383613646030426 8 Purity/1FRt0SRqqt0FBt/FRt_purity‚0.15121334791183472 ,&Purity/2FRt2SRqqt2FBt/event_proportion‚0 particle/accuracy_2_of_3‚1 epoch‚5 9"Purity/*FRt*SRqqt2FBt/SRqqt_purity‚0.1944444477558136 3"Purity/*FRt1SRqqt*FBt/event_purity‚ 0.2373046875 9"Purity/1FRt1SRqqt0FBt/SRqqt_purity‚0.4894152581691742 +particle/specificity‚0.8954885601997375 jet/accuracy_2_of_5‚1 jet/accuracy_5_of_5‚0 =&Purity/1FRt0SRqqt0FBt/event_proportion‚0.4408714771270752 @&Purity/1FRt1SRqqt2FBt/event_proportion‚0.0002325148816453293 9"Purity/2FRt1SRqqt0FBt/SRqqt_purity‚0.4932953417301178 7 Purity/*FRt*SRqqt1FBt/FRt_purity‚0.5419623851776123 8 Purity/1FRt*SRqqt*FBt/FRt_purity‚0.19006238877773285 7 Purity/1FRt0SRqqt1FBt/FRt_purity‚0.5911997556686401 8 Purity/1FRt2SRqqt*FBt/FRt_purity‚0.17419393360614777 & Purity/2FRt*SRqqt*FBt/FBt_purity‚1 @&Purity/2FRt0SRqqt1FBt/event_proportion‚0.0007672990905120969 +jet/accuracy_3_of_3‚0.04310527816414833 =&Purity/0FRt0SRqqt0FBt/event_proportion‚0.0462472103536129 ("Purity/0FRt2SRqqt1FBt/event_purity‚0 9"Purity/1FRt1SRqqt*FBt/SRqqt_purity‚0.4912567734718323 *jet/accuracy_2_of_4‚0.5311389565467834 :"Purity/*FRt*SRqqt*FBt/event_purity‚0.21763379871845245 9"Purity/*FRt*SRqqt1FBt/SRqqt_purity‚0.4870149493217468 ,&Purity/*FRt2SRqqt2FBt/event_proportion‚0 :"Purity/1FRt*SRqqt*FBt/event_purity‚0.12177232652902603 :"Purity/1FRt*SRqqt1FBt/event_purity‚0.36483657360076904 ;"Purity/1FRt1SRqqt0FBt/event_purity‚0.017476167529821396 :"Purity/1FRt2SRqqt*FBt/SRqqt_purity‚0.48529788851737976 7 Purity/2FRt*SRqqt0FBt/FRt_purity‚0.1820273995399475 =&Purity/0FRt0SRqqt*FBt/event_proportion‚0.0390159972012043 :"Purity/0FRt1SRqqt1FBt/SRqqt_purity‚0.47433504462242126 9"Purity/2FRt*SRqqt*FBt/SRqqt_purity‚0.5157805681228638 @&Purity/2FRt*SRqqt1FBt/event_proportion‚0.0016043527284637094 ("Purity/2FRt1SRqqt0FBt/event_purity‚0 particle/accuracy_1_of_1‚1 ("Purity/2FRt1SRqqt2FBt/SRqqt_purity‚0 jet/accuracy_4_of_5‚0 =&Purity/*FRt0SRqqt2FBt/event_proportion‚0.0103004090487957 9"Purity/1FRt1SRqqt2FBt/SRqqt_purity‚0.2222222238779068 ?&Purity/2FRt1SRqqt0FBt/event_proportion‚0.011928013525903225 * Purity/2FRt*SRqqt2FBt/FRt_purity‚0.625 :"Purity/2FRt*SRqqt1FBt/event_purity‚0.14500001072883606 /particle/accuracy_4_of_4‚0.9544466733932495 *particle/sensitivity‚0.580799400806427 ("Purity/*FRt1SRqqt2FBt/event_purity‚0 :"Purity/0FRt*SRqqt0FBt/event_purity‚0.37768101692199707 =&Purity/0FRt0SRqqt1FBt/event_proportion‚0.0313895083963871 :"Purity/1FRt0SRqqt0FBt/event_purity‚0.15121334791183472 particle/accuracy_2_of_4‚1 6 Purity/*FRt*SRqqt*FBt/FBt_purity‚0.985103964805603 =&Purity/*FRt0SRqqt0FBt/event_proportion‚0.4607747495174408 9"Purity/0FRt0SRqqt2FBt/event_purity‚0.9195305705070496 7 Purity/1FRt0SRqqt*FBt/FBt_purity‚0.9948703050613403 7 Purity/2FRt*SRqqt1FBt/FRt_purity‚0.5179166197776794 ("Purity/2FRt2SRqqt0FBt/event_purity‚0 ,&Purity/2FRt2SRqqt1FBt/event_proportion‚0 particle/accuracy_1_of_2‚1 =&Purity/*FRt*SRqqt1FBt/event_proportion‚0.0617094486951828 ,&Purity/0FRt2SRqqt2FBt/event_proportion‚0 particle/accuracy_3_of_5‚1 =&Purity/*FRt1SRqqt0FBt/event_proportion‚0.3670479953289032 8 Purity/2FRt2SRqqt0FBt/FRt_purity‚0.16415536403656006 7 Purity/0FRt*SRqqt1FBt/FBt_purity‚0.9910179972648621 ("Purity/1FRt2SRqqt1FBt/event_purity‚0 =&Purity/2FRt*SRqqt0FBt/event_proportion‚0.0382486991584301 9"Purity/2FRt*SRqqt0FBt/SRqqt_purity‚0.5133836269378662 *jet/accuracy_1_of_3‚0.8393213152885437 8 Purity/*FRt*SRqqt*FBt/FRt_purity‚0.19078032672405243 8 Purity/*FRt0SRqqt0FBt/FRt_purity‚0.15008629858493805 ) Purity/*FRt1SRqqt2FBt/FRt_purity‚0.75 7 Purity/1FRt*SRqqt1FBt/FRt_purity‚0.5497034788131714 7 Purity/1FRt0SRqqt*FBt/FRt_purity‚0.1644042283296585 >&Purity/1FRt2SRqqt*FBt/event_proportion‚0.02322823740541935 8 Purity/1FRt2SRqqt1FBt/FRt_purity‚0.10000000149011612 particle/accuracy_1_of_4‚1 :"Purity/*FRt1SRqqt0FBt/event_purity‚0.23486140370368958 8&Purity/1FRt2SRqqt0FBt/event_proportion‚ 0.02294921875 9"Purity/2FRt2SRqqt*FBt/SRqqt_purity‚0.5378608107566833 7 Purity/0FRt1SRqqt2FBt/FBt_purity‚0.9230769276618958 & Purity/2FRt0SRqqt1FBt/FBt_purity‚1 jet/accuracy_1_of_5‚1 ?&Purity/*FRt*SRqqt2FBt/event_proportion‚0.010997953824698925 7 Purity/*FRt0SRqqt2FBt/FRt_purity‚0.8024943470954895 7 Purity/*FRt1SRqqt0FBt/FRt_purity‚0.2578768730163574 9"Purity/*FRt1SRqqt2FBt/SRqqt_purity‚0.1944444477558136 7 Purity/*FRt0SRqqt1FBt/FRt_purity‚0.5850986838340759 :"Purity/*FRt2SRqqt*FBt/SRqqt_purity‚0.46106192469596863 7 Purity/0FRt*SRqqt2FBt/FBt_purity‚0.9566286206245422 8 Purity/1FRt*SRqqt0FBt/FRt_purity‚0.17754867672920227 8 Purity/1FRt2SRqqt0FBt/FRt_purity‚0.17547883093357086 /particle/accuracy_3_of_4‚0.9868094325065613 7 Purity/0FRt1SRqqt1FBt/FBt_purity‚0.9828576445579529 @&Purity/0FRt1SRqqt2FBt/event_proportion‚0.0004417782765813172 ("Purity/1FRt1SRqqt2FBt/event_purity‚0 ?&Purity/2FRt2SRqqt0FBt/event_proportion‚0.006417410913854837 & Purity/2FRt0SRqqt2FBt/FBt_purity‚1 8 Purity/*FRt1SRqqt1FBt/FRt_purity‚0.49162915349006653 ("Purity/*FRt2SRqqt1FBt/event_purity‚0 9"Purity/0FRt*SRqqt1FBt/event_purity‚0.8169169425964355 =&Purity/1FRt*SRqqt0FBt/event_proportion‚0.6105840802192688 7 Purity/1FRt*SRqqt2FBt/FRt_purity‚0.8137756586074829 jet/accuracy_3_of_5‚1 +jet/accuracy_1_of_1‚0.22841714322566986 ("Purity/2FRt1SRqqt2FBt/event_purity‚0 @&Purity/*FRt2SRqqt1FBt/event_proportion‚0.0009998140158131719 6 Purity/0FRt0SRqqt*FBt/FBt_purity‚0.983509361743927 :"Purity/*FRt0SRqqt0FBt/event_purity‚0.14494143426418304 6 Purity/1FRt*SRqqt*FBt/FBt_purity‚0.990930438041687 & Purity/1FRt2SRqqt*FBt/FBt_purity‚1 8 Purity/2FRt*SRqqt*FBt/FRt_purity‚0.19703145325183868 A&Purity/2FRt0SRqqt2FBt/event_proportion‚0.00018601190822664648 & Purity/2FRt1SRqqt2FBt/FBt_purity‚1 <&Purity/*FRt*SRqqt0FBt/event_proportion‚0.881045401096344 9"Purity/*FRt*SRqqt2FBt/event_purity‚0.7734366655349731 ?&Purity/0FRt1SRqqt1FBt/event_proportion‚0.011555989272892475 7 Purity/1FRt1SRqqt1FBt/FBt_purity‚0.9965986609458923 :"Purity/*FRt0SRqqt*FBt/event_purity‚0.21786430478096008 @&Purity/1FRt*SRqqt2FBt/event_proportion‚0.0027204242069274187 =&Purity/1FRt1SRqqt*FBt/event_proportion‚0.1534598171710968 ;"Purity/2FRt*SRqqt0FBt/event_purity‚0.003052480984479189 @&Purity/2FRt*SRqqt2FBt/event_proportion‚0.0002092633949359879 *jet/accuracy_1_of_4‚0.8945884704589844 +jet/accuracy_3_of_4‚0.06452339142560959 _runtime‚1669.025785986 7 Purity/*FRt1SRqqt*FBt/FBt_purity‚0.9797961711883545 :"Purity/*FRt1SRqqt1FBt/event_purity‚0.29466062784194946 :"Purity/1FRt1SRqqt1FBt/event_purity‚0.14769329130649567 =&Purity/2FRt1SRqqt*FBt/event_proportion‚0.0127883180975914 9"Purity/2FRt1SRqqt*FBt/SRqqt_purity‚0.5000112056732178 8 Purity/2FRt2SRqqt*FBt/FRt_purity‚0.16415536403656006 particle/accuracy_1_of_5‚1 =&Purity/*FRt1SRqqt*FBt/event_proportion‚0.3866024911403656 :"Purity/*FRt1SRqqt0FBt/SRqqt_purity‚0.43954604864120483 & Purity/*FRt2SRqqt1FBt/FBt_purity‚1 & Purity/2FRt1SRqqt1FBt/FBt_purity‚1 ?&Purity/2FRt2SRqqt*FBt/event_proportion‚0.006417410913854837 7 Purity/*FRt0SRqqt*FBt/FBt_purity‚0.9865747094154358 9"Purity/0FRt1SRqqt*FBt/event_purity‚0.4002392590045929 9"Purity/0FRt2SRqqt*FBt/event_purity‚0.1678280383348465 7 Purity/1FRt1SRqqt2FBt/FBt_purity‚0.7777777910232544 @&Purity/2FRt1SRqqt1FBt/event_proportion‚0.0008370535797439516 ("Purity/0FRt0SRqqt0FBt/event_purity‚1 :"Purity/*FRt2SRqqt1FBt/SRqqt_purity‚0.37037038803100586 :"Purity/0FRt*SRqqt1FBt/SRqqt_purity‚0.46391940116882324 & Purity/0FRt2SRqqt*FBt/FBt_purity‚1 =&Purity/2FRt0SRqqt*FBt/event_proportion‚0.0208565853536129 trainer/global_step‚2399 7 Purity/*FRt2SRqqt*FBt/FRt_purity‚0.1716202050447464 =&Purity/1FRt0SRqqt*FBt/event_proportion‚0.4530552327632904 >&Purity/1FRt0SRqqt1FBt/event_proportion‚0.00969587080180645 7 Purity/2FRt1SRqqt*FBt/FRt_purity‚0.2769392728805542 & Purity/2FRt*SRqqt2FBt/FBt_purity‚1 ("Purity/2FRt*SRqqt2FBt/SRqqt_purity‚0 9"Purity/1FRt*SRqqt2FBt/SRqqt_purity‚0.2222222238779068 :"Purity/*FRt*SRqqt0FBt/SRqqt_purity‚0.44474148750305176 =&Purity/*FRt1SRqqt1FBt/event_proportion‚0.0188569575548172 :"Purity/*FRt2SRqqt0FBt/event_purity‚0.07654369622468948 9"Purity/0FRt1SRqqt*FBt/SRqqt_purity‚0.4044322371482849 9"Purity/0FRt2SRqqt*FBt/SRqqt_purity‚0.4235526919364929 :"Purity/1FRt0SRqqt*FBt/event_purity‚0.16143345832824707 particle/accuracy_2_of_5‚1 8&Purity/*FRt2SRqqt0FBt/event_proportion‚ 0.05322265625 >&Purity/0FRt2SRqqt0FBt/event_proportion‚0.02385602705180645 ("Purity/1FRt2SRqqt0FBt/event_purity‚0 & Purity/2FRt*SRqqt1FBt/FBt_purity‚1 ("Purity/2FRt1SRqqt*FBt/event_purity‚0 :"Purity/0FRt1SRqqt0FBt/SRqqt_purity‚0.40133675932884216 8 Purity/1FRt1SRqqt*FBt/FRt_purity‚0.26799511909484863 :"Purity/1FRt2SRqqt1FBt/SRqqt_purity‚0.32499998807907104 :"Purity/*FRt*SRqqt0FBt/event_purity‚0.17822706699371338 9"Purity/*FRt2SRqqt0FBt/SRqqt_purity‚0.4627830982208252 7 Purity/*FRt*SRqqt2FBt/FBt_purity‚0.9623135924339294 7 Purity/1FRt0SRqqt2FBt/FBt_purity‚0.9922618865966797 ("Purity/2FRt2SRqqt*FBt/event_purity‚0 particle/accuracy_2_of_2‚1 (particle/accuracy‚0.8279117345809937 :"Purity/0FRt1SRqqt2FBt/SRqqt_purity‚0.11538461595773697 9"Purity/*FRt1SRqqt1FBt/SRqqt_purity‚0.5011158585548401 =&Purity/0FRt*SRqqt1FBt/event_proportion‚0.0436662957072258 9"Purity/1FRt1SRqqt1FBt/SRqqt_purity‚0.5441378355026245 :"Purity/1FRt2SRqqt0FBt/SRqqt_purity‚0.48749205470085144 *jet/accuracy_2_of_3‚0.3694019913673401 6validation_average_jet_accuracy‚0.2750850319862366 :"Purity/0FRt*SRqqt2FBt/SRqqt_purity‚0.11538461595773697 _step‚197 :"Purity/0FRt1SRqqt0FBt/event_purity‚0.40133675932884216 >&Purity/0FRt2SRqqt*FBt/event_proportion‚0.02457682229578495 @&Purity/2FRt1SRqqt2FBt/event_proportion‚2.325148852833081e-05 9"Purity/1FRt0SRqqt2FBt/event_purity‚0.5716742277145386 @&Purity/1FRt1SRqqt1FBt/event_proportion‚0.0064639137126505375 7 Purity/*FRt*SRqqt1FBt/FBt_purity‚0.9928315281867981 9"Purity/*FRt0SRqqt2FBt/event_purity‚0.8277560472488403 ?&Purity/0FRt*SRqqt2FBt/event_proportion‚0.008068266324698925 7 Purity/1FRt0SRqqt1FBt/FBt_purity‚0.9960317015647888 @&Purity/1FRt0SRqqt2FBt/event_proportion‚0.0024879092816263437 8 Purity/2FRt0SRqqt*FBt/FRt_purity‚0.15780822932720184 :"Purity/*FRt2SRqqt*FBt/event_purity‚0.07511558383703232 A&Purity/1FRt2SRqqt1FBt/event_proportion‚0.00027901786961592734 % _timestamp‚1.7844619946369004e+09 8 Purity/*FRt2SRqqt0FBt/FRt_purity‚0.17233282327651978 7 Purity/0FRt*SRqqt*FBt/FBt_purity‚0.9822075963020325 0"Purity/0FRt2SRqqt1FBt/SRqqt_purity‚ 0.4140625 8 Purity/1FRt1SRqqt1FBt/FRt_purity‚0.49488621950149536 8 Purity/2FRt1SRqqt0FBt/FRt_purity‚0.26527270674705505 *loss/val_total_loss‚12.930234135310835 9"Purity/0FRt0SRqqt1FBt/event_purity‚0.9938056468963623 <&Purity/2FRt*SRqqt*FBt/event_proportion‚0.040062315762043 *jet/accuracy_1_of_2‚0.6758744120597839 particle/accuracy_1_of_3‚1 & Purity/2FRt0SRqqt*FBt/FBt_purity‚1 particle/accuracy_4_of_5‚1 =&Purity/*FRt*SRqqt*FBt/event_proportion‚0.9537528157234192 ;"Purity/2FRt*SRqqt*FBt/event_purity‚0.010592108592391014 9"Purity/2FRt0SRqqt2FBt/event_purity‚0.4285714328289032 ( Purity/2FRt1SRqqt2FBt/FRt_purity‚0.5 9"Purity/*FRt*SRqqt*FBt/SRqqt_purity‚0.4461800158023834 =&Purity/0FRt1SRqqt*FBt/event_proportion‚0.2203543484210968 9"Purity/1FRt*SRqqt*FBt/SRqqt_purity‚0.4897123873233795 ("Purity/1FRt2SRqqt*FBt/event_purity‚0 <&Purity/*FRt2SRqqt*FBt/event_proportion‚0.054222472012043 :"Purity/1FRt*SRqqt0FBt/event_purity‚0.11342722922563553 8 Purity/*FRt2SRqqt1FBt/FRt_purity‚0.10000000149011612 9"Purity/0FRt*SRqqt2FBt/event_purity‚0.8659148216247559 7 Purity/0FRt1SRqqt*FBt/FBt_purity‚0.9777034521102905 8 Purity/1FRt1SRqqt0FBt/FRt_purity‚0.25692325830459595 ;"Purity/2FRt0SRqqt0FBt/event_purity‚0.005765735637396574 ("Purity/2FRt1SRqqt1FBt/event_purity‚0 7 Purity/2FRt1SRqqt1FBt/FRt_purity‚0.4629629850387573 :"Purity/2FRt0SRqqt*FBt/event_purity‚0.02024012617766857 ("Purity/0FRt1SRqqt2FBt/event_purity‚0 9"Purity/0FRt0SRqqt*FBt/event_purity‚0.9803093671798706 :"Purity/0FRt2SRqqt0FBt/event_purity‚0.17278362810611725 @&Purity/0FRt2SRqqt1FBt/event_proportion‚0.0007207961170934141 7 Purity/1FRt1SRqqt2FBt/FRt_purity‚0.7777777910232544 particle/accuracy_5_of_5‚0 ?&Purity/0FRt0SRqqt2FBt/event_proportion‚0.007626487873494625 =&Purity/0FRt1SRqqt0FBt/event_proportion‚0.2083565890789032 :"Purity/1FRt*SRqqt0FBt/SRqqt_purity‚0.48872464895248413 7 Purity/*FRt1SRqqt2FBt/FBt_purity‚0.8518518209457397 7 Purity/0FRt0SRqqt1FBt/FBt_purity‚0.9938056468963623 =&Purity/*FRt0SRqqt*FBt/event_proportion‚0.5129278302192688 :"Purity/0FRt*SRqqt*FBt/event_purity‚0.45958709716796875 =&Purity/0FRt*SRqqt*FBt/event_proportion‚0.2839471697807312 7 Purity/1FRt*SRqqt1FBt/FBt_purity‚0.9967347383499146 & Purity/1FRt2SRqqt1FBt/FBt_purity‚1 ,jet/accuracy_4_of_4‚0.013899499550461769 7 Purity/2FRt0SRqqt1FBt/FRt_purity‚0.5729166269302368 =&Purity/*FRt0SRqqt1FBt/event_proportion‚0.0418526791036129 7 Purity/*FRt0SRqqt2FBt/FBt_purity‚0.9688039422035217 9"Purity/0FRt2SRqqt0FBt/SRqqt_purity‚0.4242400527000427 7 Purity/1FRt1SRqqt*FBt/FBt_purity‚0.9835585355758667 @&Purity/*FRt1SRqqt2FBt/event_proportion‚0.0006975446594879031 =&Purity/0FRt*SRqqt0FBt/event_proportion‚0.2322126179933548 =&Purity/1FRt*SRqqt*FBt/event_proportion‚0.6297432780265808 9"Purity/1FRt*SRqqt2FBt/event_purity‚0.5321617126464844 :"Purity/1FRt1SRqqt*FBt/event_purity‚0.02296149544417858 9"Purity/2FRt*SRqqt1FBt/SRqqt_purity‚0.6481481194496155 7 Purity/*FRt1SRqqt1FBt/FBt_purity‚0.9878615140914917 7 Purity/0FRt0SRqqt2FBt/FBt_purity‚0.9597653150558472 >&Purity/1FRt*SRqqt1FBt/event_proportion‚0.01643880270421505 9"Purity/1FRt0SRqqt1FBt/event_purity‚0.5171013474464417 'particle/f_score‚0.5917399525642395 ,"Purity/2FRt*SRqqt2FBt/event_purity‚0.375 7 Purity/2FRt0SRqqt2FBt/FRt_purity‚0.6428571343421936 & Purity/*FRt2SRqqt*FBt/FBt_purity‚1 9"Purity/0FRt1SRqqt1FBt/event_purity‚0.4010334014892578 6 Purity/1FRt*SRqqt2FBt/FBt_purity‚0.976190447807312 >&Purity/2FRt0SRqqt0FBt/event_proportion‚0.01990327425301075 & Purity/2FRt1SRqqt*FBt/FBt_purity‚1 9"Purity/*FRt*SRqqt1FBt/event_purity‚0.6793755292892456 6 Purity/*FRt0SRqqt1FBt/FBt_purity‚0.994796633720398 :"Purity/*FRt1SRqqt*FBt/SRqqt_purity‚0.44203704595565796 9"Purity/1FRt*SRqqt1FBt/SRqqt_purity‚0.5291669964790344 8 Purity/2FRt0SRqqt0FBt/FRt_purity‚0.13816353678703308 +jet/accuracy_2_of_2‚0.09808406233787537 9"Purity/0FRt*SRqqt0FBt/SRqqt_purity‚0.4053645730018616 =&Purity/1FRt1SRqqt0FBt/event_proportion‚0.1467633992433548 /particle/accuracy_3_of_3‚0.9994933605194092 /"Purity/2FRt0SRqqt1FBt/event_purity‚0.296875 7 Purity/*FRt*SRqqt2FBt/FRt_purity‚0.7996599078178406 7 Purity/*FRt1SRqqt*FBt/FRt_purity‚0.2691303491592407 9"Purity/0FRt*SRqqt*FBt/SRqqt_purity‚0.4076361358165741 & Purity/0FRt2SRqqt1FBt/FBt_purity‚1 7 Purity/1FRt0SRqqt2FBt/FRt_purity‚0.8138890266418457 9"Purity/2FRt2SRqqt0FBt/SRqqt_purity‚0.5378608107566833‘1¢( î þ{ ?&Purity/*FRt*SRqqt2FBt/event_proportion‚0.010997953824698925 7 Purity/*FRt0SRqqt2FBt/FRt_purity‚0.8024943470954895 7 Purity/*FRt1SRqqt0FBt/FRt_purity‚0.2578768730163574 9"Purity/*FRt1SRqqt2FBt/SRqqt_purity‚0.1944444477558136 7 Purity/*FRt0SRqqt1FBt/FRt_purity‚0.5850986838340759 :"Purity/*FRt2SRqqt*FBt/SRqqt_purity‚0.46106192469596863 7 Purity/0FRt*SRqqt2FBt/FBt_purity‚0.9566286206245422 8 Purity/1FRt*SRqqt0FBt/FRt_purity‚0.17754867672920227 8 Purity/1FRt2SRqqt0FBt/FRt_purity‚0.17547883093357086 /particle/accuracy_3_of_4‚0.9868094325065613 7 Purity/0FRt1SRqqt1FBt/FBt_purity‚0.9828576445579529 @&Purity/0FRt1SRqqt2FBt/event_proportion‚0.0004417782765813172 ("Purity/1FRt1SRqqt2FBt/event_purity‚0 ?&Purity/2FRt2SRqqt0FBt/event_proportion‚0.006417410913854837 & Purity/2FRt0SRqqt2FBt/FBt_purity‚1 8 Purity/*FRt1SRqqt1FBt/FRt_purity‚0.49162915349006653 ("Purity/*FRt2SRqqt1FBt/event_purity‚0 9"Purity/0FRt*SRqqt1FBt/event_purity‚0.8169169425964355 =&Purity/1FRt*SRqqt0FBt/event_proportion‚0.6105840802192688 7 Purity/1FRt*SRqqt2FBt/FRt_purity‚0.8137756586074829 jet/accuracy_3_of_5‚1 +jet/accuracy_1_of_1‚0.22841714322566986 ("Purity/2FRt1SRqqt2FBt/event_purity‚0 @&Purity/*FRt2SRqqt1FBt/event_proportion‚0.0009998140158131719 6 Purity/0FRt0SRqqt*FBt/FBt_purity‚0.983509361743927 :"Purity/*FRt0SRqqt0FBt/event_purity‚0.14494143426418304 6 Purity/1FRt*SRqqt*FBt/FBt_purity‚0.990930438041687 & Purity/1FRt2SRqqt*FBt/FBt_purity‚1 8 Purity/2FRt*SRqqt*FBt/FRt_purity‚0.19703145325183868 A&Purity/2FRt0SRqqt2FBt/event_proportion‚0.00018601190822664648 & Purity/2FRt1SRqqt2FBt/FBt_purity‚1 <&Purity/*FRt*SRqqt0FBt/event_proportion‚0.881045401096344 9"Purity/*FRt*SRqqt2FBt/event_purity‚0.7734366655349731 ?&Purity/0FRt1SRqqt1FBt/event_proportion‚0.011555989272892475 7 Purity/1FRt1SRqqt1FBt/FBt_purity‚0.9965986609458923 :"Purity/*FRt0SRqqt*FBt/event_purity‚0.21786430478096008 @&Purity/1FRt*SRqqt2FBt/event_proportion‚0.0027204242069274187 =&Purity/1FRt1SRqqt*FBt/event_proportion‚0.1534598171710968 ;"Purity/2FRt*SRqqt0FBt/event_purity‚0.003052480984479189 @&Purity/2FRt*SRqqt2FBt/event_proportion‚0.0002092633949359879 *jet/accuracy_1_of_4‚0.8945884704589844 +jet/accuracy_3_of_4‚0.06452339142560959 _runtime‚1669.025785986 7 Purity/*FRt1SRqqt*FBt/FBt_purity‚0.9797961711883545 :"Purity/*FRt1SRqqt1FBt/event_purity‚0.29466062784194946 :"Purity/1FRt1SRqqt1FBt/event_purity‚0.14769329130649567 =&Purity/2FRt1SRqqt*FBt/event_proportion‚0.0127883180975914 9"Purity/2FRt1SRqqt*FBt/SRqqt_purity‚0.5000112056732178 8 Purity/2FRt2SRqqt*FBt/FRt_purity‚0.16415536403656006 particle/accuracy_1_of_5‚1 =&Purity/*FRt1SRqqt*FBt/event_proportion‚0.3866024911403656 :"Purity/*FRt1SRqqt0FBt/SRqqt_purity‚0.43954604864120483 & Purity/*FRt2SRqqt1FBt/FBt_purity‚1 & Purity/2FRt1SRqqt1FBt/FBt_purity‚1 ?&Purity/2FRt2SRqqt*FBt/event_proportion‚0.006417410913854837 7 Purity/*FRt0SRqqt*FBt/FBt_purity‚0.9865747094154358 9"Purity/0FRt1SRqqt*FBt/event_purity‚0.4002392590045929 9"Purity/0FRt2SRqqt*FBt/event_purity‚0.1678280383348465 7 Purity/1FRt1SRqqt2FBt/FBt_purity‚0.7777777910232544 @&Purity/2FRt1SRqqt1FBt/event_proportion‚0.0008370535797439516 ("Purity/0FRt0SRqqt0FBt/event_purity‚1 :"Purity/*FRt2SRqqt1FBt/SRqqt_purity‚0.37037038803100586 :"Purity/0FRt*SRqqt1FBt/SRqqt_purity‚0.46391940116882324 & Purity/0FRt2SRqqt*FBt/FBt_purity‚1 =&Purity/2FRt0SRqqt*FBt/event_proportion‚0.0208565853536129 trainer/global_step‚2399 7 Purity/*FRt2SRqqt*FBt/FRt_purity‚0.1716202050447464 =&Purity/1FRt0SRqqt*FBt/event_proportion‚0.4530552327632904 >&Purity/1FRt0SRqqt1FBt/event_proportion‚0.00969587080180645 7 Purity/2FRt1SRqqt*FBt/FRt_purity‚0.2769392728805542 & Purity/2FRt*SRqqt2FBt/FBt_purity‚1 ("Purity/2FRt*SRqqt2FBt/SRqqt_purity‚0 9"Purity/1FRt*SRqqt2FBt/SRqqt_purity‚0.2222222238779068 :"Purity/*FRt*SRqqt0FBt/SRqqt_purity‚0.44474148750305176 =&Purity/*FRt1SRqqt1FBt/event_proportion‚0.0188569575548172 :"Purity/*FRt2SRqqt0FBt/event_purity‚0.07654369622468948 9"Purity/0FRt1SRqqt*FBt/SRqqt_purity‚0.4044322371482849 9"Purity/0FRt2SRqqt*FBt/SRqqt_purity‚0.4235526919364929 :"Purity/1FRt0SRqqt*FBt/event_purity‚0.16143345832824707 particle/accuracy_2_of_5‚1 8&Purity/*FRt2SRqqt0FBt/event_proportion‚ 0.05322265625 >&Purity/0FRt2SRqqt0FBt/event_proportion‚0.02385602705180645 ("Purity/1FRt2SRqqt0FBt/event_purity‚0 & Purity/2FRt*SRqqt1FBt/FBt_purity‚1 ("Purity/2FRt1SRqqt*FBt/event_purity‚0 :"Purity/0FRt1SRqqt0FBt/SRqqt_purity‚0.40133675932884216 8 Purity/1FRt1SRqqt*FBt/FRt_purity‚0.26799511909484863 :"Purity/1FRt2SRqqt1FBt/SRqqt_purity‚0.32499998807907104 :"Purity/*FRt*SRqqt0FBt/event_purity‚0.17822706699371338 9"Purity/*FRt2SRqqt0FBt/SRqqt_purity‚0.4627830982208252 7 Purity/*FRt*SRqqt2FBt/FBt_purity‚0.9623135924339294 7 Purity/1FRt0SRqqt2FBt/FBt_purity‚0.9922618865966797 ("Purity/2FRt2SRqqt*FBt/event_purity‚0 particle/accuracy_2_of_2‚1 (particle/accuracy‚0.8279117345809937 :"Purity/0FRt1SRqqt2FBt/SRqqt_purity‚0.11538461595773697 9"Purity/*FRt1SRqqt1FBt/SRqqt_purity‚0.5011158585548401 =&Purity/0FRt*SRqqt1FBt/event_proportion‚0.0436662957072258 9"Purity/1FRt1SRqqt1FBt/SRqqt_purity‚0.5441378355026245 :"Purity/1FRt2SRqqt0FBt/SRqqt_purity‚0.48749205470085144 *jet/accuracy_2_of_3‚0.3694019913673401 6validation_average_jet_accuracy‚0.2750850319862366 :"Purity/0FRt*SRqqt2FBt/SRqqt_purity‚0.11538461595773697 _step‚197 :"Purity/0FRt1SRqqt0FBt/event_purity‚0.40133675932884216 >&Purity/0FRt2SRqqt*FBt/event_proportion‚0.02457682229578495 @&Purity/2FRt1SRqqt2FBt/event_proportion‚2.325148852833081e-05 9"Purity/1FRt0SRqqt2FBt/event_purity‚0.5716742277145386 @&Purity/1FRt1SRqqt1FBt/event_proportion‚0.0064639137126505375 7 Purity/*FRt*SRqqt1FBt/FBt_purity‚0.9928315281867981 9"Purity/*FRt0SRqqt2FBt/event_purity‚0.8277560472488403 ?&Purity/0FRt*SRqqt2FBt/event_proportion‚0.008068266324698925 7 Purity/1FRt0SRqqt1FBt/FBt_purity‚0.9960317015647888 @&Purity/1FRt0SRqqt2FBt/event_proportion‚0.0024879092816263437 8 Purity/2FRt0SRqqt*FBt/FRt_purity‚0.15780822932720184 :"Purity/*FRt2SRqqt*FBt/event_purity‚0.07511558383703232 A&Purity/1FRt2SRqqt1FBt/event_proportion‚0.00027901786961592734 % _timestamp‚1.7844619946369004e+09 8 Purity/*FRt2SRqqt0FBt/FRt_purity‚0.17233282327651978 7 Purity/0FRt*SRqqt*FBt/FBt_purity‚0.9822075963020325 0"Purity/0FRt2SRqqt1FBt/SRqqt_purity‚ 0.4140625 8 Purity/1FRt1SRqqt1FBt/FRt_purity‚0.49488621950149536 8 Purity/2FRt1SRqqt0FBt/FRt_purity‚0.26527270674705505 *loss/val_total_loss‚12.930234135310835 9"Purity/0FRt0SRqqt1FBt/event_purity‚0.9938056468963623 <&Purity/2FRt*SRqqt*FBt/event_proportion‚0.040062315762043 *jet/accuracy_1_of_2‚0.6758744120597839 particle/accuracy_1_of_3‚1 & Purity/2FRt0SRqqt*FBt/FBt_purity‚1 particle/accuracy_4_of_5‚1 =&Purity/*FRt*SRqqt*FBt/event_proportion‚0.9537528157234192 ;"Purity/2FRt*SRqqt*FBt/event_purity‚0.010592108592391014 9"Purity/2FRt0SRqqt2FBt/event_purity‚0.4285714328289032 ( Purity/2FRt1SRqqt2FBt/FRt_purity‚0.5 9"Purity/*FRt*SRqqt*FBt/SRqqt_purity‚0.4461800158023834 =&Purity/0FRt1SRqqt*FBt/event_proportion‚0.2203543484210968 9"Purity/1 ¤)$ø FRt*SRqqt*FBt/SRqqt_purity‚0.4897123873233795 ("Purity/1FRt2SRqqt*FBt/event_purity‚0 <&Purity/*FRt2SRqqt*FBt/event_proportion‚0.054222472012043 :"Purity/1FRt*SRqqt0FBt/event_purity‚0.11342722922563553 8 Purity/*FRt2SRqqt1FBt/FRt_purity‚0.10000000149011612 9"Purity/0FRt*SRqqt2FBt/event_purity‚0.8659148216247559 7 Purity/0FRt1SRqqt*FBt/FBt_purity‚0.9777034521102905 8 Purity/1FRt1SRqqt0FBt/FRt_purity‚0.25692325830459595 ;"Purity/2FRt0SRqqt0FBt/event_purity‚0.005765735637396574 ("Purity/2FRt1SRqqt1FBt/event_purity‚0 7 Purity/2FRt1SRqqt1FBt/FRt_purity‚0.4629629850387573 :"Purity/2FRt0SRqqt*FBt/event_purity‚0.02024012617766857 ("Purity/0FRt1SRqqt2FBt/event_purity‚0 9"Purity/0FRt0SRqqt*FBt/event_purity‚0.9803093671798706 :"Purity/0FRt2SRqqt0FBt/event_purity‚0.17278362810611725 @&Purity/0FRt2SRqqt1FBt/event_proportion‚0.0007207961170934141 7 Purity/1FRt1SRqqt2FBt/FRt_purity‚0.7777777910232544 particle/accuracy_5_of_5‚0 ?&Purity/0FRt0SRqqt2FBt/event_proportion‚0.007626487873494625 =&Purity/0FRt1SRqqt0FBt/event_proportion‚0.2083565890789032 :"Purity/1FRt*SRqqt0FBt/SRqqt_purity‚0.48872464895248413 7 Purity/*FRt1SRqqt2FBt/FBt_purity‚0.8518518209457397 7 Purity/0FRt0SRqqt1FBt/FBt_purity‚0.9938056468963623 =&Purity/*FRt0SRqqt*FBt/event_proportion‚0.5129278302192688 :"Purity/0FRt*SRqqt*FBt/event_purity‚0.45958709716796875 =&Purity/0FRt*SRqqt*FBt/event_proportion‚0.2839471697807312 7 Purity/1FRt*SRqqt1FBt/FBt_purity‚0.9967347383499146 & Purity/1FRt2SRqqt1FBt/FBt_purity‚1 ,jet/accuracy_4_of_4‚0.013899499550461769 7 Purity/2FRt0SRqqt1FBt/FRt_purity‚0.5729166269302368 =&Purity/*FRt0SRqqt1FBt/event_proportion‚0.0418526791036129 7 Purity/*FRt0SRqqt2FBt/FBt_purity‚0.9688039422035217 9"Purity/0FRt2SRqqt0FBt/SRqqt_purity‚0.4242400527000427 7 Purity/1FRt1SRqqt*FBt/FBt_purity‚0.9835585355758667 @&Purity/*FRt1SRqqt2FBt/event_proportion‚0.0006975446594879031 =&Purity/0FRt*SRqqt0FBt/event_proportion‚0.2322126179933548 =&Purity/1FRt*SRqqt*FBt/event_proportion‚0.6297432780265808 9"Purity/1FRt*SRqqt2FBt/event_purity‚0.5321617126464844 :"Purity/1FRt1SRqqt*FBt/event_purity‚0.02296149544417858 9"Purity/2FRt*SRqqt1FBt/SRqqt_purity‚0.6481481194496155 7 Purity/*FRt1SRqqt1FBt/FBt_purity‚0.9878615140914917 7 Purity/0FRt0SRqqt2FBt/FBt_purity‚0.9597653150558472 >&Purity/1FRt*SRqqt1FBt/event_proportion‚0.01643880270421505 9"Purity/1FRt0SRqqt1FBt/event_purity‚0.5171013474464417 'particle/f_score‚0.5917399525642395 ,"Purity/2FRt*SRqqt2FBt/event_purity‚0.375 7 Purity/2FRt0SRqqt2FBt/FRt_purity‚0.6428571343421936 & Purity/*FRt2SRqqt*FBt/FBt_purity‚1 9"Purity/0FRt1SRqqt1FBt/event_purity‚0.4010334014892578 6 Purity/1FRt*SRqqt2FBt/FBt_purity‚0.976190447807312 >&Purity/2FRt0SRqqt0FBt/event_proportion‚0.01990327425301075 & Purity/2FRt1SRqqt*FBt/FBt_purity‚1 9"Purity/*FRt*SRqqt1FBt/event_purity‚0.6793755292892456 6 Purity/*FRt0SRqqt1FBt/FBt_purity‚0.994796633720398 :"Purity/*FRt1SRqqt*FBt/SRqqt_purity‚0.44203704595565796 9"Purity/1FRt*SRqqt1FBt/SRqqt_purity‚0.5291669964790344 8 Purity/2FRt0SRqqt0FBt/FRt_purity‚0.13816353678703308 +jet/accuracy_2_of_2‚0.09808406233787537 9"Purity/0FRt*SRqqt0FBt/SRqqt_purity‚0.4053645730018616 =&Purity/1FRt1SRqqt0FBt/event_proportion‚0.1467633992433548 /particle/accuracy_3_of_3‚0.9994933605194092 /"Purity/2FRt0SRqqt1FBt/event_purity‚0.296875 7 Purity/*FRt*SRqqt2FBt/FRt_purity‚0.7996599078178406 7 Purity/*FRt1SRqqt*FBt/FRt_purity‚0.2691303491592407 9"Purity/0FRt*SRqqt*FBt/SRqqt_purity‚0.4076361358165741 & Purity/0FRt2SRqqt1FBt/FBt_purity‚1 7 Purity/1FRt0SRqqt2FBt/FRt_purity‚0.8138890266418457 9"Purity/2FRt2SRqqt0FBt/SRqqt_purity‚0.5378608107566833 ,&Purity/1FRt2SRqqt2FBt/event_proportion‚0 9"Purity/2FRt1SRqqt1FBt/SRqqt_purity‚0.6481481194496155 8"Purity/*FRt0SRqqt1FBt/event_purity‚0.871714174747467 7 Purity/*FRt*SRqqt0FBt/FRt_purity‚0.1779700070619583 8 Purity/*FRt0SRqqt*FBt/FRt_purity‚0.16383613646030426 8 Purity/1FRt0SRqqt0FBt/FRt_purity‚0.15121334791183472 ,&Purity/2FRt2SRqqt2FBt/event_proportion‚0 particle/accuracy_2_of_3‚1 epoch‚5 9"Purity/*FRt*SRqqt2FBt/SRqqt_purity‚0.1944444477558136 3"Purity/*FRt1SRqqt*FBt/event_purity‚ 0.2373046875 9"Purity/1FRt1SRqqt0FBt/SRqqt_purity‚0.4894152581691742 +particle/specificity‚0.8954885601997375 jet/accuracy_2_of_5‚1 jet/accuracy_5_of_5‚0 =&Purity/1FRt0SRqqt0FBt/event_proportion‚0.4408714771270752 @&Purity/1FRt1SRqqt2FBt/event_proportion‚0.0002325148816453293 9"Purity/2FRt1SRqqt0FBt/SRqqt_purity‚0.4932953417301178 7 Purity/*FRt*SRqqt1FBt/FRt_purity‚0.5419623851776123 8 Purity/1FRt*SRqqt*FBt/FRt_purity‚0.19006238877773285 7 Purity/1FRt0SRqqt1FBt/FRt_purity‚0.5911997556686401 8 Purity/1FRt2SRqqt*FBt/FRt_purity‚0.17419393360614777 & Purity/2FRt*SRqqt*FBt/FBt_purity‚1 @&Purity/2FRt0SRqqt1FBt/event_proportion‚0.0007672990905120969 +jet/accuracy_3_of_3‚0.04310527816414833 =&Purity/0FRt0SRqqt0FBt/event_proportion‚0.0462472103536129 ("Purity/0FRt2SRqqt1FBt/event_purity‚0 9"Purity/1FRt1SRqqt*FBt/SRqqt_purity‚0.4912567734718323 *jet/accuracy_2_of_4‚0.5311389565467834 :"Purity/*FRt*SRqqt*FBt/event_purity‚0.21763379871845245 9"Purity/*FRt*SRqqt1FBt/SRqqt_purity‚0.4870149493217468 ,&Purity/*FRt2SRqqt2FBt/event_proportion‚0 :"Purity/1FRt*SRqqt*FBt/event_purity‚0.12177232652902603 :"Purity/1FRt*SRqqt1FBt/event_purity‚0.36483657360076904 ;"Purity/1FRt1SRqqt0FBt/event_purity‚0.017476167529821396 :"Purity/1FRt2SRqqt*FBt/SRqqt_purity‚0.48529788851737976 7 Purity/2FRt*SRqqt0FBt/FRt_purity‚0.1820273995399475 =&Purity/0FRt0SRqqt*FBt/event_proportion‚0.0390159972012043 :"Purity/0FRt1SRqqt1FBt/SRqqt_purity‚0.47433504462242126 9"Purity/2FRt*SRqqt*FBt/SRqqt_purity‚0.5157805681228638 @&Purity/2FRt*SRqqt1FBt/event_proportion‚0.0016043527284637094 ("Purity/2FRt1SRqqt0FBt/event_purity‚0 particle/accuracy_1_of_1‚1 ("Purity/2FRt1SRqqt2FBt/SRqqt_purity‚0 jet/accuracy_4_of_5‚0 =&Purity/*FRt0SRqqt2FBt/event_proportion‚0.0103004090487957 9"Purity/1FRt1SRqqt2FBt/SRqqt_purity‚0.2222222238779068 ?&Purity/2FRt1SRqqt0FBt/event_proportion‚0.011928013525903225 * Purity/2FRt*SRqqt2FBt/FRt_purity‚0.625 :"Purity/2FRt*SRqqt1FBt/event_purity‚0.14500001072883606 /particle/accuracy_4_of_4‚0.9544466733932495 *particle/sensitivity‚0.580799400806427 ("Purity/*FRt1SRqqt2FBt/event_purity‚0 :"Purity/0FRt*SRqqt0FBt/event_purity‚0.37768101692199707 =&Purity/0FRt0SRqqt1FBt/event_proportion‚0.0313895083963871 :"Purity/1FRt0SRqqt0FBt/event_purity‚0.15121334791183472 particle/accuracy_2_of_4‚1 6 Purity/*FRt*SRqqt*FBt/FBt_purity‚0.985103964805603 =&Purity/*FRt0SRqqt0FBt/event_proportion‚0.4607747495174408 9"Purity/0FRt0SRqqt2FBt/event_purity‚0.9195305705070496 7 Purity/1FRt0SRqqt*FBt/FBt_purity‚0.9948703050613403 7 Purity/2FRt*SRqqt1FBt/FRt_purity‚0.5179166197776794 ("Purity/2FRt2SRqqt0FBt/event_purity‚0 ,&Purity/2FRt2SRqqt1FBt/event_proportion‚0 particle/accuracy_1_of_2‚1 =&Purity/*FRt*SRqqt1FBt/event_proportion‚0.0617094486951828 ,&Purity/0FRt2SRqqt2FBt/event_proportion‚0 particle/accuracy_3_of_5‚1 =&Purity/*FRt1SRqqt0FBt/event_proportion‚0.3670479953289032 8 Purity/2FRt2SRqqt0FBt/FRt_purity‚0.16415536403656006 7 Purity/0FRt*SRqqt1FBt/FBt_purity‚0.9910179972648621 ("Purity/1FRt2SRqqt1FBt/event_purity‚0 =&Purity/2FRt*SRqqt0FBt/event_proportion‚0.0382486991584301 9"Purity/2FRt*SRqqt0FBt/SRqqt_purity‚0.5133836269378662 *jet/accuracy_1_of_3‚0.8393213152885437 8 Purity/*FRt*SRqqt*FBt/FRt_purity‚0.19078032672405243 8 Purity/*FRt0SRqqt0FBt/FRt_purity‚0.15008629858493805 ) Purity/*FRt1SRqqt2FBt/FRt_purity‚0.75 7 Purity/1FRt*SRqqt1FBt/FRt_purity‚0.5497034788131714 7 Purity/1FRt0SRqqt*FBt/FRt_purity‚0.1644042283296585 >&Purity/1FRt2SRqqt*FBt/event_proportion‚0.02322823740541935 8 Purity/1FRt2SRqqt1FBt/FRt_purity‚0.10000000149011612 particle/accuracy_1_of_4‚1 :"Purity/*FRt1SRqqt0FBt/event_purity‚0.23486140370368958 8&Purity/1FRt2SRqqt0FBt/event_proportion‚ 0.02294921875 9"Purity/2FRt2SRqqt*FBt/SRqqt_purity‚0.5378608107566833 7 Purity/0FRt1SRqqt2FBt/FBt_purity‚0.9230769276618958 & Purity/2FRt0SRqqt1FBt/FBt_purity‚1 jet/accuracy_1_of_5‚1Å¢‘9lï ‚(:á µõòÒ‚žù¤ disk.nvme0n1.in‚ 1852.4453125! disk.nvme0n1.out‚ 21176.171875 disk.nvme1n1.in‚ 129.765625& disk.nvme1n1.out‚12180.25439453125" disk.dm-0.out‚32005.9638671875 disk.dm-1.in‚0 disk.rbd0.in‚0 disk.rbd0.out‚0) disk./.usagePercent‚76.15163977342986$ disk./.usageGB‚1169.118049621582 disk.md0.in‚0" disk.nvme0c0n1.in‚ 1852.4453125$ memory_percent‚98.82197380065918' proc.memory.availableMB‚ 193.0078125# proc.memory.rssMB‚ 2845.97265625 proc.cpu.threads‚17 disk.nvme1c1n1.in‚ 129.765625( disk.nvme1c1n1.out‚12180.25439453125" disk.dm-1.out‚1257.99951171875# disk.nvme0c0n1.out‚ 21176.171875 network.sent‚ 3.526178e+06 network.recv‚473779 disk.md0.out‚ 43.0546875 disk.dm-0.in‚ 1982.2109375) proc.memory.percent‚17.37043857574463 cpu‚25.858536271806702@N%‹žð ‚(:“ µõòÒºù€¦ gpu.0.gpu‚84.0 gpu.0.memory‚20, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚48.0 gpu.0.powerWatts‚222.392' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚74.13066666666667 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ˆ(ãdvñ ‚(:ë ÄõòÒœ¸â¤$ memory_percent‚98.82199764251709) proc.memory.percent‚17.37043857574463 disk.nvme0n1.in‚ 1852.4453125& disk.nvme0n1.out‚22696.57080078125 disk.dm-1.out‚1322.021484375 cpu‚25.857296372887575 disk.nvme1c1n1.in‚ 129.765625% disk.nvme1n1.out‚12246.9423828125 disk.md0.in‚0( proc.memory.availableMB‚ 193.00390625# proc.memory.rssMB‚ 2845.97265625 proc.cpu.threads‚17) disk./.usagePercent‚76.28191320695429% disk./.usageGB‚1171.1180725097656 disk.nvme1n1.in‚ 129.765625 disk.md0.out‚ 43.66796875 disk.dm-0.in‚ 1982.2109375# disk.dm-0.out‚33527.72021484375 disk.rbd0.in‚0( disk.nvme0c0n1.out‚22696.57080078125 network.sent‚ 3.584487e+06 network.recv‚477639' disk.nvme1c1n1.out‚12246.9423828125 disk.dm-1.in‚0 disk.rbd0.out‚0" disk.nvme0c0n1.in‚ 1852.4453125[-žò ‚(:“ ÄõòÒ웥 gpu.0.gpu‚55.0 gpu.0.memory‚10, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚48.0 gpu.0.powerWatts‚223.159' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚74.38633333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¬i2ó - _wandbruntime‚1701  _runtime‚1701”üÆÉô à ' lr-AdamW/pg1‚0.00013552144705789574 ' lr-AdamW/pg2‚0.00013552144705789574 trainer/global_step‚2449 % _timestamp‚1.7844620274466941e+09 _runtime‚1701.835068233 _step‚198³T¹ÒÎõ È trainer/global_step‚2449 % _timestamp‚1.7844620274466941e+09 _runtime‚1701.835068233 _step‚198 ' lr-AdamW/pg1‚0.00013552144705789574 ' lr-AdamW/pg2‚0.00013552144705789574Æ0ƒª¶2ö - _wandbruntime‚1701  _runtime‚1701‹–{à'÷ ÚO E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚44067742 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚19425821 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚190114792457728 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚19425823 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2015928514358 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚192136160394752 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2021473636352 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚189886955602182 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2081528790528 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚189886851776396 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚18593805 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2021384976384 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚192136266094080 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2021473636352 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚19425823 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚24652064 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚190114775418368 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚144484411711488 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚190114775418368 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2021384976384 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚142402977030656 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚142402882920960 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5555597 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚187871027087824 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚192136160394752 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚13038027 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚24641921 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚190114792457728 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚144484624779264 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚24652064 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 trainer/global_step‚2449 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 213067776 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2081647748608 % _timestamp‚1.7844620274476159e+09 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚13038031 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚181 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 _step‚199 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 _runtime‚1701.835872895 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚192136160394752 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2021473636352 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚189886955602182 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2081528790528 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚189886851776396 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚18593805 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2021384976384 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚192136266094080 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2021473636352 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚19425823 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚24652064 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚190114775418368 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚144484411711488 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚190114775418368 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2021384976384 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚142402977030656 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚142402882920960 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5555597 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚187871027087824 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚192136160394752 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚13038027 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚24641921 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚190114792457728 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚144484624779264 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚24652064 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 trainer/global_step‚2449 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 213067776 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2081647748608 % _timestamp‚1.7844620274476159e+09 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚13038031 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚181 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 _step‚199 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 _runtime‚1701.835872895 IDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚44085191 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚187945317613216 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 _runtime‚1702.244147301 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚19433450 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allo"æ“M$cated‚142459114758656 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚189962029622286 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2016625222532 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚190189942124032 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚24661898 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚24651741 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2022259359232 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚190 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2016712009070 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚44085191 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚18600779 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚190189942124032 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚19433452 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚186 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2082454722560 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚192212095776768 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚44095350 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 trainer/global_step‚2449 % _timestamp‚1.7844620278555977e+09 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚19433452 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚192212201483264 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2022170692096 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2022170692096 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚44095350 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5557696 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚24651741 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _step‚200 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚144541356420608 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚142459020648960 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360- :ï&û éM RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚24651741 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _step‚200 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚144541356420608 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚142459020648960 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚44085191 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚187945317613216 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 _runtime‚1702.244147301 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚19433450 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚142459114758656 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚189962029622286 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2016625222532 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚190189942124032 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚24661898 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚24651741 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2022259359232 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚190 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2016712009070 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚44085191 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚18600779 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚190189942124032 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚19433452 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚186 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2082454722560 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚192212095776768 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚44095350 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 trainer/global_step‚2449 % _timestamp‚1.7844620278555977e+09 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚19433452 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚192212201483264 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2022170692096 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2022170692096 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚44095350 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5557696 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0È9ª,ä2ü - _wandbruntime‚1702  _runtime‚1702»cý ÿ 0loss/FRt1/assignment_loss‚2.9221163573033655 1loss/FBt1/assignment_loss‚0.10292286518320302 1loss/SRqqt1/detection_loss‚1.3229642868503382 trainer/global_step‚2449 _runtime‚1702.244679502 1loss/SRqqt1/assignment_loss‚0.818604473456091 2loss/SRqqt2/assignment_loss‚0.7932382408618287 1loss/FBt2/assignment_loss‚0.26123560077510777 /loss/FRt2/detection_loss‚1.3744540200585535 /loss/FBt2/detection_loss‚1.7286749281873925 &loss/total_loss‚13.270241453813878 epoch‚6 _step‚201 /loss/FRt1/detection_loss‚0.5785743183065858 .loss/FBt1/detection_loss‚1.146530080001282 0loss/FRt2/assignment_loss‚1.4960474962015882 0loss/SRqqt2/detection_loss‚0.724878786628542 % _timestamp‚1.7844620278565478e+09Ï C• þ „ 0loss/FRt1/assignment_loss‚2.9221163573033655 1loss/FBt1/assignment_loss‚0.10292286518320302 1loss/SRqqt1/detection_loss‚1.3229642868503382 trainer/global_step‚2449 _runtime‚1702.244679502 1loss/SRqqt1/assignment_loss‚0.818604473456091 2loss/SRqqt2/assignment_loss‚0.7932382408618287 1loss/FBt2/assignment_loss‚0.26123560077510777 /loss/FRt2/detection_loss‚1.3744540200585535 /loss/FBt2/detection_loss‚1.7286749281873925 &loss/total_loss‚13.270241453813878 epoch‚6 _step‚201 /loss/FRt1/detection_loss‚0.5785743183065858 .loss/FBt1/detection_loss‚1.146530080001282 0loss/FRt2/assignment_loss‚1.4960474962015882 0loss/SRqqt2/detection_loss‚0.724878786628542 % _timestamp‚1.7844620278565478e+09É8u!oÿ ‚(:ä ÓõòÒԨᤠdisk.md0.out‚ 44.14453125 disk.rbd0.out‚0( disk.nvme0c0n1.out‚22697.69970703125% proc.memory.availableMB‚ 192.96875# proc.memory.rssMB‚ 2845.97265625) disk./.usagePercent‚76.33160204593104% disk./.usageGB‚1171.8809204101562 disk.dm-0.in‚ 1982.2109375# disk.dm-0.out‚33528.63330078125 disk.nvme0n1.in‚ 1852.4453125& disk.nvme0n1.out‚22697.69970703125$ memory_percent‚98.82221221923828 proc.cpu.threads‚17 disk.nvme1n1.in‚ 129.765625 disk.md0.in‚0 disk.dm-1.out‚1322.048828125 disk.rbd0.in‚0" disk.nvme0c0n1.in‚ 1852.4453125& disk.nvme1c1n1.out‚12247.763671875 network.sent‚ 3.58751e+06$ disk.nvme1n1.out‚12247.763671875 disk.dm-1.in‚0 disk.nvme1c1n1.in‚ 129.765625) proc.memory.percent‚17.37043857574463 cpu‚25.85563757553462 network.recv‚481103j¥Ôž€ ‚(:“ ÓõòÒÿ¡Ä¥ gpu.0.gpu‚59.0 gpu.0.memory‚10, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚49.0 gpu.0.powerWatts‚224.305' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚74.76833333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0k&"™g ‚(:Ü âõòÒóÊô¤! disk./.usageGB‚1171.880859375 disk.md0.out‚ 44.4765625& disk.nvme0n1.out‚23107.40966796875) proc.memory.percent‚17.37043857574463 proc.cpu.threads‚17) disk./.usagePercent‚76.33159807034686 disk.md0.in‚0 disk.dm-1.in‚0 disk.rbd0.out‚0( disk.nvme0c0n1.out‚23107.40966796875 disk.nvme0n1.in‚ 1852.4453125 disk.nvme1c1n1.in‚ 129.765625 disk.nvme1n1.in‚ 129.765625 network.sent‚ 3.649959e+06# disk.dm-0.out‚33938.34228515625 disk.rbd0.in‚0" disk.nvme0c0n1.in‚ 1852.4453125# disk.nvme1c1n1.out‚ 12248.609375% proc.memory.availableMB‚ 192.96875# proc.memory.rssMB‚ 2845.97265625 cpu‚25.857885050915424 network.recv‚484950 disk.dm-0.in‚ 1982.2109375! disk.dm-1.out‚1322.1826171875! disk.nvme1n1.out‚ 12248.609375$ memory_percent‚98.82221221923828^™ÁŸ‚ ‚(:” âõòұล gpu.0.gpu‚93.0 gpu.0.memory‚22, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚47.0 gpu.0.powerWatts‚113.889' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚37.962999999999994 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚01¾"2ƒ - _wandbruntime‚1734  _runtime‚1734s\ƒÚÉ„ à ' lr-AdamW/pg2‚0.00013533936021086157 trainer/global_step‚2499 % _timestamp‚1.7844620599099288e+09 _runtime‚1734.298432541 _step‚202 ' lr-AdamW/pg1‚0.00013533936021086157aØVÎ… È ' lr-AdamW/pg1‚0.00013533936021086157 ' lr-AdamW/pg2‚0.00013533936021086157 trainer/global_step‚2499 % _timestamp‚1.7844620599099288e+09 _runtime‚1734.298432541 _step‚202Ê­ƒý2† - _wandbruntime‚1734  _runtime‚1734ñã.y"‡ ÙO YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚193872258733568 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚18955039 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2055080258740 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚18955213 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚19807273 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2055167045166 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 _step‚203 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚193872275772928 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚195932993650176 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚19807273 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚25144916 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚5664382 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚147331948062720 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2122178741760 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2060734916608 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚145209863430656 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚19807271 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚191585536318064 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 $ _timestamp‚1.784462059914162e+09 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚44942044 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚19807271 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚174 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚145209769320960 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2060823576576 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚191585553357424 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚13290831 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚195932993650176 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚25134773 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚13290827 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚193640720402590 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚195933099349504 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2060823576576 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 UGDeviceStatsMonitor.on_train_batch_start‡ÔO£Ì/allocated_bytes.small_pool.peak‚ 269300736 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚170 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5664212 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚25134773 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 110569472 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚195933099349504 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚25144916 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 _runtime‚1734.302558121M!g¯ä'ˆ ÞO H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚13290831 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚195932993650176 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚25134773 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚13290827 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚193640720402590 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚195933099349504 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2060823576576 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚170 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5664212 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚25134773 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 110569472 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚195933099349504 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚25144916 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 _runtime‚1734.302558121 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚193872258733568 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚18955039 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2055080258740 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚18955213 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚19807273 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2055167045166 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 _step‚203 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚193872275772928 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚195932993650176 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚19807273 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚25144916 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚5664382 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚147331948062720 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2122178741760 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2060734916608 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚145209863430656 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚19807271 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚191585536318064 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 $ _timestamp‚1.784462059914162e+09 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚44942044 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚19807271 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚174 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚145209769320960 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2060823576576 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚191585553357424ËH/˜F2‰ - _wandbruntime‚1734  _runtime‚1734ºö ê&Š äM M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚196009034738688 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚19814902 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚44969652 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚25154750 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚5666570 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚145266001158656 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2061520632320 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚19814902 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚2499 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚44959493 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚193947425439232 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚196009034738688 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚19814900 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 206769152 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚44969652 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2061520632320 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2061609299456 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚25144593 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚18962457 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚193947408399872 % _timestamp‚1.7844620603211422e+09 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚176 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚147388898407936 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚193947408399872 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚196008929032192 _step‚204 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5666394 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚18962277 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 _runtime‚1734.709712097 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚193715690596796 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚25154750 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚180 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚191659843882816 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2061609299456 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚147389105177088 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2055950539878 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2123104018432 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚44959493C,ÊÙï&‹ éM S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚196008929032192 _step‚204 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5666394 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚18962277 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 _runtime‚1734.709712097 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚193715690596796 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚25154750 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚180 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚191659843882816 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2061609299456 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚147389105177088 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2055950539878 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2123104018432 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚44959493 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚196009034738688 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚19814902 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚44969652 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚25154750 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚5666570 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚145266001158656 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2061520632320 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚19814902 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚2499 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚44959493 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚193947425439232 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚196009034738688 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚19814900 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 206769152 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚44969652 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2061520632320 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2061609299456 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚25144593 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚18962457 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚193947408399872 % _timestamp‚1.7844620603211422e+09 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚176 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚147388898407936 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚193947408399872ÌÔ¶3™2Œ - _wandbruntime‚1734  _runtime‚1734‚ú"d € 0loss/FRt2/assignment_loss‚1.2706960607977476 /loss/FRt1/detection_loss‚0.5993684233757157 /loss/FRt2/detection_loss‚1.3699240701849367 /loss/FBt2/detection_loss‚1.5906797661057983 trainer/global_step‚2499 % _timestamp‚1.7844620603228054e+09 2loss/SRqqt2/assignment_loss‚0.7375939663245006 1loss/SRqqt1/detection_loss‚1.5529106175482417 _runtime‚1734.710875859 /loss/FRt1/assignment_loss‚2.936484896369766 2loss/SRqqt1/assignment_loss‚0.8054670832815032 0loss/FBt1/assignment_loss‚0.1405890187547667 1loss/SRqqt2/detection_loss‚0.7267765029069769 &loss/total_loss‚13.007345530764894 1loss/FBt2/assignment_loss‚0.19761971597563388 /loss/FBt1/detection_loss‚1.0792354091393057 epoch‚6 _step‚205º!BÕŽ … /loss/FRt1/detection_loss‚0.5993684233757157 /loss/FRt2/detection_loss‚1.3699240701849367 /loss/FBt2/detection_loss‚1.5906797661057983 trainer/global_step‚2499 % _timestamp‚1.78446206032ÌøE628054e+09 2loss/SRqqt2/assignment_loss‚0.7375939663245006 1loss/SRqqt1/detection_loss‚1.5529106175482417 _runtime‚1734.710875859 /loss/FRt1/assignment_loss‚2.936484896369766 2loss/SRqqt1/assignment_loss‚0.8054670832815032 0loss/FBt1/assignment_loss‚0.1405890187547667 1loss/SRqqt2/detection_loss‚0.7267765029069769 &loss/total_loss‚13.007345530764894 1loss/FBt2/assignment_loss‚0.19761971597563388 /loss/FBt1/detection_loss‚1.0792354091393057 epoch‚6 _step‚205 0loss/FRt2/assignment_loss‚1.2706960607977476Í»½›1t ‚(:é ñõòҰ̼¤ cpu‚25.859399437859516% disk./.usageGB‚1171.8337516784668 disk.rbd0.in‚0" disk.nvme0c0n1.in‚ 1852.4453125 disk.nvme0n1.in‚ 1852.4453125 disk.md0.in‚0 disk.md0.out‚ 44.74609375# proc.memory.rssMB‚ 2845.97265625) proc.memory.percent‚17.37043857574463& disk.nvme0n1.out‚23847.78271484375& disk.nvme1c1n1.out‚12376.095703125 disk.nvme1n1.in‚ 129.765625 proc.cpu.threads‚17 network.sent‚ 3.654778e+06 disk.dm-0.in‚ 1982.2109375# disk.dm-0.out‚34805.49462890625 disk.dm-1.in‚0! disk.dm-1.out‚1322.2841796875( disk.nvme0c0n1.out‚23847.78271484375$ memory_percent‚98.82264137268066' proc.memory.availableMB‚ 192.8984375 network.recv‚488038) disk./.usagePercent‚76.32852966477556 disk.rbd0.out‚0 disk.nvme1c1n1.in‚ 129.765625$ disk.nvme1n1.out‚12376.095703125yòﬓ ‚(:ˆ ñõòÒ‘¦¥ gpu.0.gpu‚92.0 gpu.0.memory‚20, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚47.0 gpu.0.powerWatts‚113.829' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚37.943 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0 ¨‘Òž‘ ‚(:“ €öòÒäÜߤ gpu.0.gpu‚47.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚47.0 gpu.0.powerWatts‚113.755' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚37.91833333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0‘uÌs’ ‚(:è €öòÒ‡¹ù¤ disk.nvme1c1n1.in‚ 129.765625' disk.nvme1c1n1.out‚12377.7900390625 network.recv‚492421) disk./.usagePercent‚76.15148596801659' proc.memory.availableMB‚ 192.6484375# proc.memory.rssMB‚ 2845.97265625 network.sent‚ 3.718509e+06 disk.md0.out‚ 45.04296875 disk.dm-0.in‚ 1982.2109375 disk.nvme1n1.in‚ 129.765625) proc.memory.percent‚17.37043857574463 proc.cpu.threads‚17% disk./.usageGB‚1169.1156883239746 disk.md0.in‚0 disk.dm-1.in‚0 disk.dm-1.out‚ 1322.41015625& disk.nvme0n1.out‚23852.61474609375% disk.nvme1n1.out‚12377.7900390625$ memory_percent‚98.82416725158691 cpu‚25.85746739302041# disk.dm-0.out‚34811.24267578125 disk.rbd0.in‚0 disk.rbd0.out‚0" disk.nvme0c0n1.in‚ 1852.4453125( disk.nvme0c0n1.out‚23852.61474609375 disk.nvme0n1.in‚ 1852.4453125Œ¢•n2“ - _wandbruntime‚1766  _runtime‚1766;yÏÖÉ” à trainer/global_step‚2549 % _timestamp‚1.7844620919952204e+09 _runtime‚1766.383626439 _step‚206 ' lr-AdamW/pg1‚0.00013515305241532284 ' lr-AdamW/pg2‚0.00013515305241532284ÙFΕ È % _timestamp‚1.7844620919952204e+09 _runtime‚1766.383626439 _step‚206 ' lr-AdamW/pg1‚0.00013515305241532284 ' lr-AdamW/pg2‚0.00013515305241532284 trainer/global_step‚2549Î;>±2– - _wandbruntime‚1766  _runtime‚1766$µpoà'— ÚO JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2094318789548 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚150179636485632 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚199729826905600 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚199729932604928 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2100173516800 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚45826491 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚25637768 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚20188721 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚199729826905600 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚195300062587664 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2100173516800 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚20188723 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚148016749830656 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 116860928 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚20188721 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2100084856832 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚19315815 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚197394485202998 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚25627625 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5772188 % _timestamp‚1.7844620919962435e+09 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚5772354 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2162769794048 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚19315985 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚166 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚25627625xíIOå'˜ ßO JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2094318789548 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚150179636485632 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚199729826905600 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚199729932604928 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2100173516800 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚45826491 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚25637768 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚20188721 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚199729826905600 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚195300062587664 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2100173516800 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚20188723 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚148016749830656 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 116860928 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚20188721 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2100084856832 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚19315815 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚197394485202998 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚25627625 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5772188 % _timestamp‚1.7844620919962435e+09 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚5772354 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2162769794048 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚19315985 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚166 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚25627625Ïõ—³ 2™ - _wandbruntime‚1766  _runtime‚1766¼³Øe!š äM [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 116853760 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2100959239680 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5774374 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚199805867994112 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚45843954 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚150236583022592 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚45833795 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚20196352 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚182 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚195374353113056 trainer/global_step‚2549 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚45833795 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚19323057 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2100959239680 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚197704908754432 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚197469559223102 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2095102284148 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚197704908754432 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚20196350 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚13548687 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚150236372059136 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 _runtime‚1766.793818005 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚148072887558656 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚148072793448960 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚45843954 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚197704891715072 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚199805867994112 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚25637445 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2095189070686 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚199805762287616 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130hfv=ï&› éM NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚25647602 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2100870572544 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚25637445 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2095189070686 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚199805762287616 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 116853760 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2100959239680 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5774374 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚199805867994112 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚45843954 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚150236583022592 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚45833795 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚20196352 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚182 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚195374353113056 trainer/global_step‚2549 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚45833795 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚19323057 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2100959239680 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚197704908754432 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚197469559223102 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2095102284148 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚197704908754432 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚20196350 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚13548687 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚150236372059136 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 _runtime‚1766.793818005 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚148072887558656 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚148072793448960 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚45843954 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚197704891715072 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚199805867994112 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 F›¢ ‚(:’ žöòÒÆí»¤ gpu.0.gpu‚91.0 gpu.0.memory‚17, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚47.0 gpu.0.powerWatts‚113.98' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚37.99333333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Ë5é)2£ - _wandbruntime‚1798  _runtime‚1798þĥƤ À & lr-AdamW/pg1‚0.0001349625356374811 & lr-AdamW/pg2‚0.0001349625356374811 trainer/global_step‚2599 $ _timestamp‚1.784462124391466e+09 _runtime‚1798.780809767 _step‚210Ks½Ë¥ Å $ _timestamp‚1.784462124391466e+09 _runtime‚1798.780809767 _step‚210 & lr-AdamW/pg1‚0.0001349625356374811 & lr-AdamW/pg2‚0.0001349625356374811 trainer/global_step‚2599ÒW¬Bö2¦ - _wandbruntime‚1798  _runtime‚1798 ›Rà'§ ÚO [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2139523457024 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2139523457024 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚174 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2133644106782 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚19677273 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2203445872640 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 _step‚211 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 % _timestamp‚1.7844621243921163e+09 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚201387225363968 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚46700793 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 trainer/global_step‚2599 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚26130620 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚203526660161024 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 _runtime‚1798.780899137 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2139434797056 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚26120477 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 129443840 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚203526765860352 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2139434797056 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚150823542120960 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2133557320356 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚26120477 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚20570171 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚203526660161024 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 _runtime‚1798.780899137 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2139434797056 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚26120477 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 129443840 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚203526765860352 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2139434797056 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚150823542120960 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2133557320356 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚26120477 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚20570171 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2133644106782 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚19677273 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2203445872640 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 _step‚211 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 % _timestamp‚1.7844621243921163e+09 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚201387225363968 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚46700793 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 trainer/global_step‚2599 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 H>DeviceStatsMonitor.on_train_baªÃ,yðtch_start/allocation.all.current‚10145 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚26130620 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚186 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚201462375030272 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚26140454 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚19684367 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚199088879382656 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚201223324023510 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚26130297 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 $ _timestamp‚1.784462124804622e+09 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚20577802 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2134427601494 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2140220512768 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚203602701249536 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 223546368 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚46708097 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚201462375030272 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚190 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚5882880 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚150879679848960 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚20577802 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2140220512768 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2140309179904 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚46718256 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚19684177 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚153083806535680 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2204126686720 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚20577800 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5882694 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2140309179904 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚13801487 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2204256123392 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚203602701249536 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚199088896422016 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 _runtime‚1799.193193982 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 129436672 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248)îv“î&« èM J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚20577800 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚46718256 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚19684177 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚153083806535680 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2204126686720 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚20577800 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5882694 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2140309179904 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚13801487 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2204256123392 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚203602701249536 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚199088896422016 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 _runtime‚1799.193193982 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 129436672 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚150879773958656 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚186 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚201462375030272 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚26140454 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚19684367 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚199088879382656 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚201223324023510 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚26130297 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 $ _timestamp‚1.784462124804622e+09 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚20577802 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2134427601494 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2140220512768 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚203602701249536 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 223546368 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚46708097 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚201462375030272 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚190 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚5882880 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚150879679848960 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚20577802 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2140220512768 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2140309179904 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E ¹ ÚO \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚205144708679168 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2172882637590 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚155874526025728 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚26613329 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚153630428520960 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚204902014803814 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚205144725718528 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2243884436992 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚207323599115776 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚205144708679168 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚20951623 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 % _timestamp‚1.7844621563146832e+09 _step‚215 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 KADeviceStxí{G¢atsMonitor.on_train_batch_start/active.small_pool.current‚10143 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2178784737280 _runtime‚1830.704560452 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5987585 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚204901910978028 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚202729132166224 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚20951623 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚207323599115776 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚5987762 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚205144725718528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚153630522630656 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚47575095 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚177 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2244003395072 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2178873397248 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚20036993 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚14049227 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚26623472 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚207323493416448 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 trainer/global_step‚2649 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚20951621 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2178784737280 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 213067776 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2172795851164 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚26613329 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚181 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚155874312957952 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚207323493416448 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚26623472 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚202729115126864 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚47564950 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303™o å'º ßO _step‚215 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2178784737280 _runtime‚1830.704560452 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚5987585 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚204901910978028 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚202729132166224 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚20951623 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚207323599115776 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚5987762 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚205144725718528 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚153630522630656 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚47575095 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚177 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2244003395072 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2178873397248 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚20036993 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚14049227 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚26623472 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚207323493416448 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 trainer/global_step‚2649 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚20951621 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2178784737280 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 213067776 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2172795851164 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚26613329 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚181 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚155874312957952 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚207323493416448 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚26623472 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚202729115126864 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚47564950 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚205144708679168 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2172882637590 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚155874526025728 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚26613329 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚153630428520960 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚204902014803814 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚205144725718528 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2243884436992 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚207323599115776 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚205144708679168 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚20951623 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 % _timestamp‚1.7844621563146832e+09×nÎ}2» - _wandbruntime‚1831  _runtime‚1831@é&¼ ãM J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚26623149 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚207399428798464 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚20959252 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚26623149 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5989685 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 219352064 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 125242368 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 trainer/global_step‚2649 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚207399534504960 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2173579345764 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 $ _timestamp‚1.784462156717419e+09 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚204976984998020 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚202803422691616 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚20043968 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚47592558 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚205219858345472 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚205219875384832 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚20959250 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚20044136 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚168 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2179659120128 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2179570452992 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚14054283 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚47582399 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚47582399 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚26633306 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚155931264220672 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 _runtime‚1831.105969385 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 _step‚216 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚207399534504960 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2173666132302 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚205219875384832 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2244823214080 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚20959250 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚47592558 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2244697971712 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚164 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚155931483572736 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚202803405652256 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚204977088823918 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚20959252 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚205219858345472 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚26633306 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0☕á;½ èM NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚26633306 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚26623149 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚207399428798464 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚20959252 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚26623149 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚5989685 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 219352064 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 125242368 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 trainer/global_step‚2649 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚207399534504960 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2173579345764 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 $ _timestamp‚1.784462156717419e+09 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚204976984998020 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚202803422691616 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚20043968 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚47592558 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚205219858345472 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚205219875384832 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚20959250 OBDeviceStatsMonitor.on_train_batch_ÓSo³end/inactive_split.all.allocated‚20044136 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚168 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2179659120128 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2179570452992 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚14054283 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚47582399 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚47582399 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚26633306 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚155931264220672 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 _runtime‚1831.105969385 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 _step‚216 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚207399534504960 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2173666132302 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚205219875384832 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2244823214080 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚20959250 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚47592558 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2244697971712 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚164 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚155931483572736 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚202803405652256 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚204977088823918 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚20959252 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚205219858345472 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682ؘ÷e¢2¾ - _wandbruntime‚1831  _runtime‚1831³Ð=•¿ € /loss/FRt1/detection_loss‚0.5658494285622899 0loss/FRt1/assignment_loss‚2.9497698307968676 2loss/SRqqt1/assignment_loss‚0.7955086428647566 2loss/SRqqt2/assignment_loss‚0.7575967112542712 0loss/FBt2/assignment_loss‚0.2572012727240936 .loss/FBt1/detection_loss‚1.079841119137073 /loss/FBt2/detection_loss‚2.0829056010891995 epoch‚6 _step‚217 &loss/total_loss‚13.730080624049577 % _timestamp‚1.7844621567183435e+09 /loss/FRt2/detection_loss‚1.5165573504854042 1loss/SRqqt1/detection_loss‚1.3054567611563885 1loss/SRqqt2/detection_loss‚0.8370371301667765 trainer/global_step‚2649 _runtime‚1831.106470156 0loss/FRt2/assignment_loss‚1.4928146087741672 1loss/FBt1/assignment_loss‚0.08954216703829018¼›Å À … 0loss/FRt2/assignment_loss‚1.4928146087741672 1loss/FBt1/assignment_loss‚0.08954216703829018 /loss/FRt1/detection_loss‚0.5658494285622899 0loss/FRt1/assignment_loss‚2.9497698307968676 2loss/SRqqt1/assignment_loss‚0.7955086428647566 2loss/SRqqt2/assignment_loss‚0.7575967112542712 0loss/FBt2/assignment_loss‚0.2572012727240936 .loss/FBt1/detection_loss‚1.079841119137073 /loss/FBt2/detection_loss‚2.0829056010891995 epoch‚6 _step‚217 &loss/total_loss‚13.730080624049577 % _timestamp‚1.7844621567183435e+09 /loss/FRt2/detection_loss‚1.5165573504854042 1loss/SRqqt1/detection_loss‚1.3054567611563885 1loss/SRqqt2/detection_loss‚0.8370371301667765 trainer/global_step‚2649 _runtime‚1831.106470156ÙÁeŸŽvÁ ‚(:ë ÚöòÒª»è¤( disk.nvme0c0n1.out‚23932.37548828125' disk.nvme1c1n1.out‚14350.4521484375 disk.md0.out‚ 47.18359375# memory_percent‚98.8243818283081( proc.memory.availableMB‚ 192.61328125# proc.memory.rssMB‚ 2845.97265625% disk./.usageGB‚1171.9915008544922 disk.dm-0.in‚ 1982.2109375 disk.dm-1.in‚0 disk.nvme1n1.in‚ 129.765625 network.recv‚514492# disk.dm-0.out‚36858.26220703125 disk.nvme0n1.in‚ 1852.4453125) proc.memory.percent‚17.37043857574463 cpu‚25.843926626622224 proc.cpu.threads‚17 network.sent‚ 3.861487e+06! disk.dm-1.out‚1323.1826171875 disk.rbd0.out‚0" disk.nvme0c0n1.in‚ 1852.4453125& disk.nvme0n1.out‚23932.37548828125 disk.nvme1c1n1.in‚ 129.765625% disk.nvme1n1.out‚14350.4521484375 disk.md0.in‚0) disk./.usagePercent‚76.33880481058408 disk.rbd0.in‚07Mjyž ‚(:“ ÚöòÒ¡ÌÐ¥ gpu.0.gpu‚91.0 gpu.0.memory‚18, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚48.0 gpu.0.powerWatts‚221.345' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚73.78166666666667 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0éu’à ‚(:‡ éöòÒÚäɤ gpu.0.gpu‚50.0 gpu.0.memory‚8, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚49.0 gpu.0.powerWatts‚198.075' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚66.025 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ºæIMpÄ ‚(:å éöòÒÝñ¥& disk.nvme1n1.out‚14377.25244140625# disk.dm-0.out‚37077.74267578125 disk.rbd0.out‚0 disk.nvme0n1.in‚ 1852.4453125 disk.nvme1c1n1.in‚ 129.765625( disk.nvme1c1n1.out‚14377.25244140625 disk.md0.out‚ 47.50390625$ memory_percent‚98.82595539093018( proc.memory.availableMB‚ 192.35546875% disk./.usageGB‚1171.7424049377441! disk.dm-1.out‚1323.2294921875$ disk.nvme0c0n1.out‚ 24125.8046875 disk.nvme1n1.in‚ 129.765625 disk.md0.in‚0) proc.memory.percent‚17.37043857574463 cpu‚25.84170103207963 proc.cpu.threads‚17 network.recv‚518517 disk.dm-0.in‚ 1982.2109375 disk.dm-1.in‚0 disk.rbd0.in‚0" disk.nvme0c0n1.in‚ 1852.4453125# proc.memory.rssMB‚ 2845.97265625 network.sent‚ 3.924976e+06) disk./.usagePercent‚76.32257970608983" disk.nvme0n1.out‚ 24125.8046875›2ò2Å - _wandbruntime‚1862  _runtime‚1862ʵòÉÆ Ã ' lr-AdamW/pg1‚0.00013456892435058414 ' lr-AdamW/pg2‚0.00013456892435058414 trainer/global_step‚2699 % _timestamp‚1.7844621885008337e+09 _runtime‚1862.889316418 _step‚218iÝ•ÎÇ È _runtime‚1862.889316418 _step‚218 ' lr-AdamW/pg1‚0.00013456892435058414 ' lr-AdamW/pg2‚0.00013456892435058414 trainer/global_step‚2699 % _timestamp‚1.7844621885008337e+09Ú‘o~Œ2È - _wandbruntime‚1862  _runtime‚1862ŽÎÃYà'É ÚO KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚208902191994368 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2218223337472 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚6094112 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚206443641396464 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2284453207552 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚27116324 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚208655779604222 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚156437409030656 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2212034381972 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚27106181 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 _runtime‚1862.889921598 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2218134677504 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚208655675778436 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2212121168398 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚211120432371200 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚183 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2218223337472 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚6094291 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚211120326671872 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚208902209033728 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 % _timestamp‚1.7844621885016534e+09 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚21333071 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚48449397 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚20396322 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚14302031 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚179 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚27116324 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚21333073aŒ½få'Ê ßO LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚20396139 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚211120432371200 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚208902191994368 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚156437314920960 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚48439252 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2218134677504 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2218134677504 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚208655675778436 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2212121168398 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚211120432371200 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚183 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2218223337472 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚6094291 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚211120326671872 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚208902209033728 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 % _timestamp‚1.7844621885016534e+09 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚21333071 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚48449397 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚20396322 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚14302031 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚179 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚27116324 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚21333073 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚208902191994368 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2218223337472 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚6094112 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚206443641396464 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2284453207552 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚27116324 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚208655779604222 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚156437409030656 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2212034381972 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚27106181 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 _runtime‚1862.889921598 IDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚48466860 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚171 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 trainer/global_step‚2699 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚208977358700032 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚27126158 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚211196262053888 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2218920393216 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚156493452648960 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚206517948961216 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2219009060352 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚175 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2285388158976 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚ 1863.29712916 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2285267110912 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_p!—…ˆWool.current‚ 121048064 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚14307083 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚158778719759872 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚208977341660672 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 215157760 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚156493546758656 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚208977358700032 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚21340700 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚208730749798428 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2218920393216 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚27116001 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚208730853624326 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚48456701 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚208977341660672 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6096263 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚27126158 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚48456701 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚21340702 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚20403346 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚211196367760384 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚21340700 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚20403521 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚6096434 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2212904663110 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2219009060352 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚21340702 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚158778934917632 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚206517931921856 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚27116001 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 _step‚220•Uë¾î&Í èM E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 _step‚220 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚48466860 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚48466860 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚171 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 trainer/global_step‚2699 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚208977358700032 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚27126158 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚211196262053888 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2218920393216 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚156493452648960 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚206517948961216 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2219009060352 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚175 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2285388158976 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚ 1863.29712916 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2285267110912 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 121048064 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚14307083 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚158778719759872 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚208977341660672 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 215157760 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚156493546758656 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚208977358700032 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚21340700 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚208730749798428 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2218920393216 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚27116001 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚208730853624326 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚48456701 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚208977341660672 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6096263 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚27126158 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚48456701 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚21340702 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚20403346 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚211196367760384 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚21340700 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚20403521 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚6096434 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2212904663110 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2219009060352 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚21340702 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚158778934917632 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚206517931921856 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚27116001Ü©Š, 2Î - _wandbruntime‚1863  _runtime‚1863Š)aÏ ý 0loss/FRt1/assignment_loss‚2.8171002991930787 2loss/SRqqt1/assignment_loss‚0.9571012650131991 1loss/FBt1/assignment_loss‚0.10841147990173641 trainer/global_step‚2699 1loss/FBt2/assignment_loss‚0.23913889009306713 .loss/FRt1/detection_loss‚0.597177408533869 0loss/SRqqt2/detection_loss‚0.740577601551103 epoch‚6 % _timestamp‚1.7844621889099967e+09 %loss/total_loss‚12.17405591130759 0loss/FRt2/assignment_loss‚1.4539152994413858 2loss/SRqqt2/assignment_loss‚0.7824608567128409 0loss/SRqqt1/detection_loss‚1.418372780515876 /loss/FBt1/detection_loss‚0.8452782906881887 /loss/FBt2/detection_loss‚0.7736683091401749 _step‚221 /loss/FRt2/detection_loss‚1.4408534305230698 _runtime‚ 1863.29811183¨%²Ð ‚ %loss/total_loss‚12.17405591130759 0loss/FRt2/assignment_loss‚1.4539152994413858 2loss/SRqqt2/assignment_loss‚0.7824608567128409 0loss/SRqqt1/detection_loss‚1.418372780515876 /loss/FBt1/detection_loss‚0.8452782906881887 /loss/FBt2/detection_loss‚0.7736683091401749 _step‚221 /loss/FRt2/detection_loss‚1.4408534305230698 _runtime‚ 1863.29811183 0loss/FRt1/assignment_loss‚2.8171002991930787 2loss/SRqqt1/assignment_loss‚0.9571012650131991 1loss/FBt1/assignment_loss‚0.10841147990173641 trainer/global_step‚2699 1loss/FBt2/assignment_loss‚0.23913889009306713 .loss/FRt1/detection_loss‚0.597177408533869 0loss/SRqqt2/detection_loss‚0.740577601551103 epoch‚6 % _timestamp‚1.7844621889099967e+09Ýa.¼àsÑ ‚(:è øöòÒË©©¥" disk.nvme0c0n1.in‚ 1852.4453125$ disk.nvme0n1.out‚24575.556640625 disk.md0.out‚ 47.8984375 disk.dm-0.in‚ 1982.2109375! disk.dm-1.out‚1323.3291015625 disk.rbd0.in‚0) proc.memory.percent‚17.37043857574463 disk.md0.in‚0 disk.nvme0n1.in‚ 1852.4453125 disk.nvme1c1n1.in‚ 129.765625( disk.nvme1c1n1.out‚14584.47119140625 disk.nvme1n1.in‚ 129.765625( proc.memory.availableMB‚ 192.56640625 proc.cpu.threads‚17 network.sent‚ 3.941686e+06 network.recv‚521941 disk.rbd0.out‚0$ memory_percent‚98.82466793060303# proc.memory.rssMB‚ 2845.97265625) disk./.usagePercent‚76.32029051501944$ disk./.usageGB‚1171.707260131836& disk.nvme0c0n1.out‚24575.556640625& disk.nvme1n1.out‚14584.47119140625# disk.dm-0.out‚37733.76611328125 disk.dm-1.in‚0 cpu‚25.843214292964433LˆWÒ ‚(:’ øöòÒí•â¥ gpu.0.gpu‚55.0 gpu.0.memory‚9, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚49.0 gpu.0.powerWatts‚225.301' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚75.10033333333332 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0MÏ‹œžÓ ‚(:“ ‡÷òÒ‚÷Ó¤ gpu.0.gpu‚49.0 gpu.0.memory‚10, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚49.0 gpu.0.powerWatts‚220.264' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚73.42133333333334 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¢ú–5pÔ ‚(:å ‡÷òÒº¶Þ¤% disk.nvme1c1n1.out‚14584.74609375 cpu‚25.841445661287096 network.sent‚ 3.991161e+06 network.recv‚525816% disk./.usageGB‚1171.7072448730469 disk.dm-0.in‚ 1982.2109375 disk.rbd0.in‚0' proc.memory.availableMB‚ 192.3203125# proc.memory.rssMB‚ 2845.97265625) proc.memory.percent‚17.37043857574463 disk.dm-1.in‚0! disk.dm-1.out‚1323.3642578125 disk.rbd0.out‚0 disk.nvme1n1.in‚ 129.765625 disk.md0.in‚0 disk.nvme1c1n1.in‚ 129.765625# disk.nvme1n1.out‚14584.74609375 disk.md0.out‚ 48.0390625$ memory_percent‚98.82616996765137 proc.cpu.threads‚17( disk./.usagePercent‚76.3202895211234# disk.dm-0.out‚37734.32861328125" disk.nvme0c0n1.in‚ 1852.4453125( disk.nvme0c0n1.out‚24576.20263671875 disk.nvme0n1.in‚ 1852.4453125& disk.nvme0n1.out‚24576.20263671875oì@Ì2Õ - _wandbruntime‚1894  _runtime‚1894ÓÅW¯ÇÖ Á _step‚222 & lr-AdamW/pg1‚0.0001343658551224464 & lr-AdamW/pg2‚0.0001343658551224464 trainer/global_step‚2749 % _timestamp‚1.7844622206024423e+09 _runtime‚1894.990825099¸ã-ÐÌ× Æ _runtime‚1894.990825099 _step‚222 & lr-AdamW/pg1‚0.0001343658551224464 & lr-AdamW/pg2‚0.0001343658551224464 trainer/global_step‚2749 % _timestamp‚1.7844622206024423e+09Þí ²2Ø - _wandbruntime‚1894  _runtime‚1894zIxß'Ù ÙO L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚21714521 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚21714523 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2257573277696 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚20755924 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚214917265626624 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚212659675309568 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚159244295430656 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 123152384 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚27609176 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 % _timestamp‚1.7844622206033998e+09 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2325118768128 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚210158167666064 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚212659692348928 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚20755735 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚14554827 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 _runtime‚ 1894.99174043 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚27609176 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚214917159927296 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2251272912780 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚189 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2251359699206 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 _step‚223 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚185 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚161569414198784 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚159244201320960 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚49323699 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚6201093 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚49313554 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0ð¸(!Ú ÞO IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚20755924 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated·£ Ã%‚214917265626624 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚212659675309568 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚159244295430656 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 123152384 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚27609176 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 % _timestamp‚1.7844622206033998e+09 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2325118768128 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚210158167666064 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚212659692348928 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚20755735 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚14554827 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 _runtime‚ 1894.99174043 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚27609176 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚214917159927296 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2251272912780 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚189 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2251359699206 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 _step‚223 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚185 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚161569414198784 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚159244201320960 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚49323699 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚6201093 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚49313554 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚21714521 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚21714523 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2257573277696 IDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚182 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚27619010 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 129436672 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 223546368 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚159300339048960 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚210232458191456 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚14559883 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚212484514598836 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2258270333440 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚212734842015232 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2258359000576 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚20763073 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚159300433158656 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚20762891 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚21722152 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6203008 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2325799844352 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 % _timestamp‚1.7844622210047424e+09 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚214993201015808 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚49341162 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚49331003 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚21722150 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 trainer/global_step‚2749 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚212734824975872 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚21722150 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚212734824975872 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2252143193918 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚27608853 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚161626362439680 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚212734842015232 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚27619010 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚20762891 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚21722152 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6203008 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2325799844352 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 % _timestamp‚1.7844622210047424e+09 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚214993201015808 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚49341162 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚49331003 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚21722150 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 trainer/global_step‚2749 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚212734824975872 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚21722150 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚212734824975872 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2252143193918 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚27608853 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚161626362439680 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚212734842015232 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚27619010 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚182 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚27619010 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 129436672 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 223546368 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚159300339048960 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚210232458191456 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚14559883 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚212484514598836 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2258270333440 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚212734842015232 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2258359000576 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚20763073 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚159300433158656 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1àÕý^52Þ - _wandbruntime‚1895  _runtime‚1895ÊUKÚß þ 0loss/FRt2/assignment_loss‚1.3613317415065562 1loss/FBt2/assignment_loss‚0.15230014662899408 trainer/global_step‚2749 % _timestamp‚1.7844622210056868e+09 0loss/FRt1/assignment_loss‚2.6969665673218275 /loss/FRt2/detection_loss‚1.3864853551716125 1loss/SRqqt2/detection_loss‚0.7451852660924719 /loss/FBt2/detection_loss‚1.2906392295844853 &loss/total_loss‚12.234701266231937 epoch‚6 _runtime‚ 1895.39379116 _step‚225 2loss/SRqqt1/assignment_loss‚0.8209512037430129 0loss/SRqqt2/assignment_loss‚0.78053523974712 /loss/FRt1/detection_loss‚0.5418057491487911 0loss/SRqqt1/detection_loss‚1.319136635577648 /loss/FBt1/detection_loss‚0.9770382024708822 1loss/FBt1/assignment_loss‚0.16232592923853617iÈ¡Î à ƒ 0loss/FRt2/assignment_loss‚1.3613317415065562 1loss/FBt2/assignment_loss‚0.15230014662899408 trainer/global_step‚2749 % _timestamp‚1.7844622210056868e+09 0loss/FRt1/assignment_loss‚2.6969665673218275 /loss/FRt2/detection_loss‚1.3864853551716125 1loss/SRqqt2/detection_loss‚0.7451852660924719 /loss/FBt2/detection_loss‚1.2906392295844853 &loss/total_loss‚12.234701266231937 epoch‚6 _runtime‚ 1895.39379116 _step‚225 2loss/SRqqt1/assignment_loss‚0.8209512037430129 0loss/SRqqt2/assignment_loss‚0.78053523974712 /loss/FRt1/detection_loss‚0.5418057491487911 0loss/SRqqt1/detection_loss‚1.319136635577648 /loss/FBt1/detection_loss‚0.9770382024708822 1loss/FBt1/assignment_loss‚0.16232592923853617ár#Þ‰“á ‚(:ˆ –÷òÒ¶Ÿÿ¤ gpu.0.gpu‚92.0 gpu.0.memory‚18, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚47.0 gpu.0.powerWatts‚114.816' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚38.272 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0–ºÆÁiâ ‚(:Þ –÷òÒûÔ•¥ disk.rbd0.in‚0 disk.rbd0.out‚0% disk.nvme0c0n1.out‚24591.78515625& disk.nvme1c1n1.out‚14586.541015625$ memory_percent‚98.82674217224121' proc.memory.availableMB‚ 192.2265625 disk.dm-1.out‚1323.462890625" disk.nvme0c0n1.in‚ 1852.4453125 disk.md0.out‚ 48.3828125# proc.memory.rssMB‚ 2845.97265625 cpu‚25.839675965217197 proc.cpu.threads‚17 disk.nvme0n1.in‚ 1852.4453125 network.recv‚528649) disk./.usagePercent‚76.32029349670758% disk./.usageGB‚1171.7073059082031 disk.nvme1n1.in‚ 129.765625 disk.md0.in‚0) proc.memory.percent‚17.37043857574463 network.sent‚ 4.008661e+06 disk.dm-1.in‚0# disk.nvme0n1.out‚24591.78515625 disk.nvme1c1n1.in‚ 129.765625$ disk.nvme1n1.out‚14586.541015625 disk.dm-0.in‚ 1982.2109375 disk.dm-0.out‚37750.87109375~Ѳ±ã ‚(:è ¥÷òÒñç¤& disk.nvme0n1.out‚24661.49169921875 disk.dm-0.in‚ 1982.2109375( proc.memory.availableMB‚ 192.53515625# proc.memory.rssMB‚ 2845.97265625)à¦Â proc.memory.percent‚17.37043857574463 network.sent‚ 4.059321e+06 network.recv‚532464# disk./.usageGB‚1172.06489944458" disk.nvme0c0n1.in‚ 1852.4453125( disk.nvme1c1n1.out‚14587.36474609375 disk.nvme1n1.in‚ 129.765625& disk.nvme1n1.out‚14587.36474609375 disk.md0.out‚ 48.62890625( disk.nvme0c0n1.out‚24661.49169921875 disk.nvme1c1n1.in‚ 129.765625 disk.dm-0.out‚37820.72265625 disk.dm-1.out‚1323.603515625 disk.rbd0.in‚0 disk.rbd0.out‚0 cpu‚25.838214394331043 proc.cpu.threads‚17 disk.nvme0n1.in‚ 1852.4453125 disk.md0.in‚0 disk.dm-1.in‚0$ memory_percent‚98.82485866546631) disk./.usagePercent‚76.34358569904445{C œä ‚(:‘ ¥÷òҼǜ¥ gpu.0.gpu‚9.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚47.0 gpu.0.powerWatts‚113.596' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚37.86533333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0 ÅÃ12å - _wandbruntime‚1926  _runtime‚1926P¨ÆkÇæ Á & lr-AdamW/pg1‚0.0001341586274722168 & lr-AdamW/pg2‚0.0001341586274722168 trainer/global_step‚2799 % _timestamp‚1.7844622524051104e+09 _runtime‚1926.794075819 _step‚226uqnÌç Æ _runtime‚1926.794075819 _step‚226 & lr-AdamW/pg1‚0.0001341586274722168 & lr-AdamW/pg2‚0.0001341586274722168 trainer/global_step‚2799 % _timestamp‚1.7844622524051104e+09â"1O2è - _wandbruntime‚1926  _runtime‚1926…®ègß'é ÙO VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚218714098882048 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2365459692544 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚22095973 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 227747840 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚21115244 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚164416547413504 trainer/global_step‚2799 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚28102028 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚164416775161344 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2296834557952 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚22095971 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚185 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚6307428 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚218714098882048 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚28091885 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 _runtime‚ 1926.79437462 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚216417175664128 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚28102028 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚162051181830656 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2296923217920 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 _step‚227 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2296834557952 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚189 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚216163205379252 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚213872710975024 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚216163309205038 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚21115055 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚50198001 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 133638144 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2290511443588 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚14807631 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚14807627 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚162051087720960 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚218713993182720 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2365593330688 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 % _timestamp‚1.7844622524058752e+09 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682Ô¸ ä'ê ÞO WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚22095973 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2290598230014 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚213872693935664 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚50187856 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚216417158624768 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚162051181830656 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2296923217920 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 _step‚227 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2296834557952 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚189 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚216163205379252 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚213872710975024 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚216163309205038 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚21115055 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚50198001 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 133638144 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2290511443588 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚14807631 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚14807627 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚162051087720960 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚218713993182720 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2365593330688 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 % _timestamp‚1.7844622524058752e+09 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚218714098882048 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2365459692544 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚22095973 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 227747840 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚21115244 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚164416547413504 trainer/global_step‚2799 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚28102028 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚164416775161344 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2296834557952 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚22095971 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚185 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚6307428 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚218714098882048 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚28091885 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 _runtime‚ 1926.79437462 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚216417175664128 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚28102028 IDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚50215464 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚162107319558656 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2291381724726 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚50215464 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚218790034271232 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚213946984461056 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚22103602 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚216492308291072 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚14812683 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚218790034271232 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚216492325330432 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚216238383225142 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚28111862 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 _step‚228 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚28101705 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 223546368 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚28101705 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚6309777 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚21122464 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 129436672 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚22103600 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚21122275 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚162107225448960 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2366396478976 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2297708940800 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚185 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2297620273664 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚22103600 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚216492308291072 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚50205305 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 trainer/global_step‚2799 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚218789928564736 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 _runtime‚1927.205306614 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚50205305 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚14812687 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚213947001500416 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚189 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6309592 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚164473716037632 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2291294938188 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2297708940800 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚216238279399244 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0.yÌí éM L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚189 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6309592 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚164473716037632 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2291294äª/Cp$938188 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2297708940800 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚216238279399244 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚50215464 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚162107319558656 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2291381724726 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚50215464 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚218790034271232 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚213946984461056 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚22103602 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚216492308291072 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚14812683 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚218790034271232 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚216492325330432 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚216238383225142 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚28111862 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 _step‚228 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚28101705 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 223546368 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚28101705 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚6309777 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚21122464 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 129436672 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚22103600 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚21122275 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚162107225448960 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2366396478976 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2297708940800 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚185 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2297620273664 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚22103600 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚216492308291072 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚50205305 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 trainer/global_step‚2799 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚218789928564736 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 _runtime‚1927.205306614 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚50205305 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚14812687 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚213947001500416äÔÝÈ2î - _wandbruntime‚1927  _runtime‚1927äðįï  2loss/SRqqt2/assignment_loss‚0.7563336439609467 1loss/FBt1/assignment_loss‚0.12504047417466568 /loss/FBt1/detection_loss‚0.7771347905356578 % _timestamp‚1.7844622528177412e+09 /loss/FRt2/assignment_loss‚1.395705924382512 trainer/global_step‚2799 _step‚229 0loss/FRt1/assignment_loss‚2.8737935874784903 2loss/SRqqt1/assignment_loss‚0.8930980904120952 1loss/FBt2/assignment_loss‚0.24560298550667262 epoch‚6 _runtime‚1927.205828434 /loss/FRt1/detection_loss‚0.5472232375342443 /loss/FRt2/detection_loss‚1.3830190635647301 1loss/SRqqt1/detection_loss‚1.4707749291788788 1loss/SRqqt2/detection_loss‚0.7602613806487212 /loss/FBt2/detection_loss‚2.0198166127033295 &loss/total_loss‚13.247804720080945­ˆ ð † /loss/FRt2/detection_loss‚1.3830190635647301 1loss/SRqqt1/detection_loss‚1.4707749291788788 1loss/SRqqt2/detection_loss‚0.7602613806487212 /loss/FBt2/detection_loss‚2.0198166127033295 &loss/total_loss‚13.247804720080945 2loss/SRqqt2/assignment_loss‚0.7563336439609467 1loss/FBt1/assignment_loss‚0.12504047417466568 /loss/FBt1/detection_loss‚0.7771347905356578 % _timestamp‚1.7844622528177412e+09 /loss/FRt2/assignment_loss‚1.395705924382512 trainer/global_step‚2799 _step‚229 0loss/FRt1/assignment_loss‚2.8737935874784903 2loss/SRqqt1/assignment_loss‚0.8930980904120952 1loss/FBt2/assignment_loss‚0.24560298550667262 epoch‚6 _runtime‚1927.205828434 /loss/FRt1/detection_loss‚0.5472232375342443åd¶QXnñ ‚(:ã ´÷òÒ›âÔ¤ proc.cpu.threads‚17% disk./.usageGB‚1171.1341667175293 disk.nvme1c1n1.in‚ 129.765625 disk.dm-0.in‚ 1982.2109375 disk.rbd0.in‚0 disk.rbd0.out‚0( proc.memory.availableMB‚ 210.73046875 network.recv‚535902% disk.nvme1n1.out‚14641.4599609375" disk.nvme0c0n1.in‚ 1852.4453125# memory_percent‚98.7138032913208# proc.memory.rssMB‚ 2827.56640625' disk.nvme1c1n1.out‚14641.4599609375 disk.nvme1n1.in‚ 129.765625 disk.md0.in‚0 disk.md0.out‚ 49.06640625! disk.dm-0.out‚38002.044921875 disk.dm-1.in‚0! disk.dm-1.out‚1323.9306640625 disk.nvme0n1.in‚ 1852.4453125 network.sent‚ 4.062344e+06) disk./.usagePercent‚76.28296151880993% disk.nvme0c0n1.out‚24789.98046875# disk.nvme0n1.out‚24789.98046875* proc.memory.percent‚17.258095741271973 cpu‚25.848445668698750/徜ò ‚(:‘ ´÷òÒøï·¥ gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚47.0 gpu.0.powerWatts‚112.936' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚37.64533333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Ef ã2ó - _wandbruntime‚1941  _runtime‚1941Å^£`Ù=ô Ó{ 7 Purity/1FRt1SRqqt1FBt/FRt_purity‚0.4525213837623596 * Purity/2FRt*SRqqt2FBt/FRt_purity‚0.625 7 Purity/*FRt1SRqqt1FBt/FRt_purity‚0.4562568664550781 9"Purity/*FRt1SRqqt1FBt/SRqqt_purity‚0.5725939869880676 & Purity/*FRt2SRqqt*FBt/FBt_purity‚1 8"Purity/0FRt1SRqqt*FBt/event_purity‚0.574561357498169 9"Purity/1FRt*SRqqt0FBt/event_purity‚0.1184980720281601 A&Purity/2FRt0SRqqt2FBt/event_proportion‚0.00018601190822664648 +jet/accuracy_3_of_4‚0.06511863321065903 trainer/global_step‚2799 9"Purity/*FRt*SRqqt*FBt/event_purity‚0.2648735046386719 @&Purity/*FRt2SRqqt1FBt/event_proportion‚0.0009998140158131719 7 Purity/0FRt0SRqqt1FBt/FBt_purity‚0.9922443628311157 =&Purity/0FRt1SRqqt0FBt/event_proportion‚0.2083565890789032 9"Purity/0FRt2SRqqt*FBt/SRqqt_purity‚0.5392657518386841 8 Purity/1FRt2SRqqt0FBt/FRt_purity‚0.09434112161397934 <&Purity/*FRt2SRqqt*FBt/event_proportion‚0.054222472012043 ("Purity/0FRt2SRqqt1FBt/event_purity‚0 9"Purity/1FRt*SRqqt2FBt/SRqqt_purity‚0.3333333432674408 :"Purity/2FRt*SRqqt1FBt/event_purity‚0.07000000029802322 jet/accuracy_1_of_5‚1 9"Purity/*FRt*SRqqt0FBt/SRqqt_purity‚0.6356189250946045 =&Purity/*FRt*SRqqt1FBt/event_proportion‚0.0617094486951828 ) Purity/*FRt1SRqqt2FBt/FRt_purity‚0.75 7 Purity/1FRt*SRqqt0FBt/FRt_purity‚0.1419883817434311 :"Purity/1FRt1SRqqt*FBt/event_purity‚0.03964243829250336 8"Purity/2FRt2SRqqt*FBt/SRqqt_purity‚0.759257972240448 /particle/accuracy_3_of_3‚0.9994933605194092 7 Purity/1FRt0SRqqt1FBt/FRt_purity‚0.5917165279388428 7 Purity/1FRt1SRqqt2FBt/FRt_purity‚0.7777777910232544 7 Purity/2FRt*SRqqt1FBt/FRt_purity‚0.4820833206176758 7 Purity/*FRt0SRqqt*FBt/FRt_purity‚0.1657065600156784 :"Purity/*FRt1SRqqt0FBt/event_purity‚0.34413379430770874 :"Purity/1FRt*SRqqt*FBt/event_purity‚0.12698841094970703 A&Purity/1FRt2SRqqt1FBt/event_proportion‚0.00027901786961592734 9"Purity/2FRt*SRqqt0FBt/SRqqt_purity‚0.7366169691085815 ?&Purity/2FRt2SRqqt*FBt/event_proportion‚0.006417410913854837 & Purity/2FRt0SRqqt*FBt/FBt_purity‚1 7 Purity/*FRt0SRqqt1FBt/FBt_purity‚0.9935435056686401 :"Purity/*FRt2SRqqt1FBt/SRqqt_purity‚0.35648149251937866 =&Purity/*FRt1SRqqt1FBt/event_proportion‚0.0188569575548172 7 Purity/0FRt0SRqqt2FBt/FBt_purity‚0.9631829261779785 =&Purity/2FRt0SRqqt*FBt/event_proportion‚0.0208565853536129 @&Purity/2FRt1SRqqt2FBt/event_proportion‚2.325148852833081e-05 8 Purity/*FRt0SRqqt0FBt/FRt_purity‚0.15204337239265442 ?&Purity/0FRt1SRqqt1FBt/event_proportion‚0.011555989272892475 =&Purity/1FRt*SRqqt0FBt/event_proportion‚0.6105840802192688 9"Purity/1FRt*SRqqt0FBt/SRqqt_purity‚0.7030450105667114 ,&Purity/1FRt2SRqqt2FBt/event_proportion‚0 =&Purity/0FRt*SRqqt1FBt/event_proportion‚0.0436662957072258 & Purity/0FRt2SRqqt1FBt/FBt_purity‚1 >&Purity/1FRt*SRqqt1FBt/event_proportion‚0.01643880270421505 7 Purity/1FRt*SRqqt1FBt/FBt_purity‚0.9967347383499146 ;"Purity/2FRt0SRqqt*FBt/event_purity‚0.025742391124367714 particle/accuracy_2_of_2‚1 7 Purity/*FRt*SRqqt1FBt/FRt_purity‚0.5231402516365051 ("Purity/*FRt2SRqqt1FBt/event_purity‚0 7 Purity/1FRt*SRqqt*FBt/FRt_purity‚0.1552247703075409 :"Purity/*FRt*SRqqt0FBt/event_purity‚0.22861839830875397 +"Purity/*FRt1SRqqt2FBt/SRqqt_purity‚0.25 9"Purity/0FRt*SRqqt*FBt/SRqqt_purity‚0.5721141695976257 & Purity/1FRt2SRqqt1FBt/FBt_purity‚1 ;"Purity/2FRt*SRqqt*FBt/event_purity‚0.013760234229266644 @&Purity/2FRt*SRqqt1FBt/event_proportion‚0.0016043527284637094 +jet/accuracy_3_of_3‚0.05433806777000427 7 Purity/*FRt0SRqqt2FBt/FBt_purity‚0.9719924926757812 :"Purity/*FRt1SRqqt1FBt/event_purity‚0.34146004915237427 @&Purity/1FRt*SRqqt2FBt/event_proportion‚0.0027204242069274187 >&Purity/1FRt2SRqqt*FBt/event_proportion‚0.02322823740541935 7 Purity/1FRt2SRqqt*FBt/FRt_purity‚0.0938316211104393 *loss/val_total_loss‚12.278507548993472 7 Purity/*FRt*SRqqt2FBt/FRt_purity‚0.8267573118209839 9"Purity/*FRt1SRqqt0FBt/SRqqt_purity‚0.6393875479698181 ?&Purity/0FRt*SRqqt2FBt/event_proportion‚0.008068266324698925 ;"Purity/2FRt*SRqqt0FBt/event_purity‚0.008214597590267658 6validation_average_jet_accuracy‚0.3298458456993103 :"Purity/0FRt*SRqqt2FBt/SRqqt_purity‚0.19230769574642181 /"Purity/0FRt2SRqqt1FBt/SRqqt_purity‚0.390625 =&Purity/1FRt1SRqqt0FBt/event_proportion‚0.1467633992433548 9"Purity/2FRt*SRqqt1FBt/SRqqt_purity‚0.7129629254341125 8&Purity/*FRt2SRqqt0FBt/event_proportion‚ 0.05322265625 >&Purity/0FRt2SRqqt*FBt/event_proportion‚0.02457682229578495 9"Purity/*FRt2SRqqt*FBt/SRqqt_purity‚0.6179335713386536 *jet/accuracy_1_of_3‚0.8990447521209717 ,"Purity/2FRt*SRqqt2FBt/event_purity‚0.375 ( Purity/2FRt1SRqqt2FBt/FRt_purity‚0.5 ,jet/accuracy_4_of_4‚0.013899499550461769 & Purity/2FRt0SRqqt1FBt/FBt_purity‚1 7 Purity/*FRt0SRqqt*FBt/FBt_purity‚0.9869740009307861 8 Purity/2FRt0SRqqt*FBt/FRt_purity‚0.15635846555233002 =&Purity/*FRt0SRqqt*FBt/event_proportion‚0.5129278302192688 8 Purity/*FRt2SRqqt*FBt/FRt_purity‚0.09250184893608093 9"Purity/0FRt1SRqqt0FBt/event_purity‚0.5831332206726074 ("Purity/1FRt2SRqqt*FBt/event_purity‚0 7 Purity/2FRt1SRqqt1FBt/FRt_purity‚0.4513888955116272 & Purity/*FRt2SRqqt1FBt/FBt_purity‚1 @&Purity/1FRt1SRqqt1FBt/event_proportion‚0.0064639137126505375 :"Purity/0FRt1SRqqt2FBt/SRqqt_purity‚0.19230769574642181 particle/accuracy_4_of_5‚1 7 Purity/0FRt0SRqqt*FBt/FBt_purity‚0.9836359620094299 ;"Purity/1FRt1SRqqt0FBt/event_purity‚0.032493408769369125 ("Purity/1FRt2SRqqt0FBt/event_purity‚0 9"Purity/1FRt1SRqqt1FBt/SRqqt_purity‚0.6245261430740356 ("Purity/2FRt1SRqqt1FBt/event_purity‚0 ("Purity/*FRt1SRqqt2FBt/event_purity‚0 :"Purity/1FRt0SRqqt*FBt/event_purity‚0.16305947303771973 :"Purity/1FRt0SRqqt1FBt/event_purity‚0.48606768250465393 9"Purity/2FRt1SRqqt0FBt/SRqqt_purity‚0.7159510254859924 9"Purity/*FRt*SRqqt1FBt/SRqqt_purity‚0.5510120987892151 8"Purity/0FRt2SRqqt0FBt/event_purity‚0.309317022562027 ?&Purity/2FRt2SRqqt0FBt/event_proportion‚0.006417410913854837 particle/accuracy_2_of_4‚1 7"Purity/0FRt*SRqqt2FBt/event_purity‚0.87195885181427 ("Purity/0FRt0SRqqt0FBt/event_purity‚1 8 Purity/2FRt1SRqqt0FBt/FRt_purity‚0.23232655227184296 jet/accuracy_5_of_5‚0 :"Purity/*FRt2SRqqt*FBt/event_purity‚0.13411442935466766 :"Purity/*FRt2SRqqt0FBt/event_purity‚0.13668189942836761 7 Purity/0FRt1SRqqt*FBt/FBt_purity‚0.9789730906486511 =&Purity/1FRt*SRqqt*FBt/event_proportion‚0.6297432780265808 =&Purity/2FRt1SRqqt*FBt/event_proportion‚0.0127883180975914 )jet/accuracy_1_of_2‚0.752110481262207 9"Purity/0FRt*SRqqt1FBt/SRqqt_purity‚0.5201519727706909 8 Purity/2FRt2SRqqt*FBt/FRt_purity‚0.08588450402021408 :"Purity/1FRt2SRqqt1FBt/SRqqt_purity‚0.32499998807907104 ?&Purity/2FRt1SRqqt0FBt/event_proportion‚0.011928013525903225 *jet/accuracy_1_of_4‚0.9432007670402527 9"Purity/2FRt0SRqqt1FBt/event_purity‚0.1822916716337204 _step‚230 6 Purity/0FRt*SRqqt2FBt/FBt_purity‚0.958831787109375 >&Purity/0FRt2SRqqt0FBt/event_proportion‚0.02385602705180645 9"Purity/1FRt2SRqqt*FBt/SRqqt_purity‚0.6652831435203552 & Purity/2FRt1SRqqt2FBt/FBt_purity‚1 7 Purity/*FRt*SRqqt*FBt/FBt_purity‚0.9852825403213501 +particle/specificity‚0.8996248841285706 8 Purity/*FRt1SRqqt*FBt/FRt_purity‚0.14784999191761017 6 Purity/0FRt*SRqqt*FBt/FBt_purity‚0.982567310333252 7 Purity/1FRt1SRqqt1FBt/FBt_purity‚0.9965986609458923 & Purity/2FRt1SRqqt*FBt/FBt_purity‚1 7 Purity/*FRt*SRqqt2FBt/FBt_purity‚0.9641268849372864 =&Purity/*FRt1SRqqt*FBt/event_proportion‚0.3866024911403656 7 Purity/*FRt1SRqqt1FBt/FBt_purity‚0.9889274835586548 =&Purity/0FRt0SRqqt*FBt/event_proportion‚0.0390159972012043 6 Purity/1FRt1SRqqt*FBt/FRt_purity‚0.131504088640213 particle/accuracy_2_of_3‚1 ("Purity/0FRt1SRqqt2FBt/event_purity‚0 7 Purity/*FRt0SRqqt2FBt/FRt_purity‚0.8370748162269592 9"Purity/0FRt1SRqqt1FBt/SRqqt_purity‚0.5403477549552917 =&Purity/1FRt1SRqqt*FBt/event_proportion‚0.1534598171710968 @&Purity/2FRt*SRqqt2FBt/event_proportion‚0.0002092633949359879 8 Purity/1FRt0SRqqt*FBt/FRt_purity‚0.16644589602947235 8 Purity/2FRt*SRqqt*FBt/FRt_purity‚0.17520101368427277 & Purity/2FRt*SRqqt1FBt/FBt_purity‚1 8 Purity/*FRt*SRqqt0FBt/FRt_purity‚0.14393380284309387 ,&Purity/*FRt2SRqqt2FBt/event_proportion‚0 9"Purity/0FRt*SRqqt0FBt/event_purity‚0.5545304417610168 9"Purity/1FRt*SRqqt1FBt/SRqqt_purity‚0.6057761311531067 7 Purity/1FRt0SRqqt1FBt/FBt_purity‚0.9960317015647888 ("Purity/2FRt2SRqqt0FBt/event_purity‚0 (particle/accuracy‚0.8357862830162048 9"Purity/0FRt*SRqqt*FBt/event_purity‚0.6061073541641235 >&Purity/1FRt0SRqqt1FBt/event_proportion‚0.00969587080180645 9"Purity/2FRt*SRqqt*FBt/SRqqt_purity‚0.7340572476387024 particle/accuracy_1_of_2‚1 7 Purity/0FRt1SRqqt2FBt/FBt_purity‚0.9038461446762085 ("Purity/2FRt*SRqqt2FBt/SRqqt_purity‚0 particle/accuracy_3_of_5‚1 =&Purity/*FRt*SRqqt*FBt/event_proportion‚0.9537528157234192 ?&Purity/*FRt*SRqqt2FBt/event_proportion‚0.010997953824698925 7 Purity/*FRt0SRqqt1FBt/FRt_purity‚0.5799595713615417 =&Purity/0FRt*SRqqt0FBt/event_proportion‚0.2322126179933548 <&Purity/2FRt*SRqqt*FBt/event_proportion‚0.040062315762043 9"Purity/*FRt*SRqqt*FBt/SRqqt_purity‚0.6315377354621887 :"Purity/0FRt2SRqqt*FBt/event_purity‚0.30029502511024475 :"Purity/1FRt*SRqqt1FBt/event_purity‚0.36711403727531433 ("Purity/2FRt1SRqqt2FBt/event_purity‚0 jet/accuracy_2_of_5‚1 particle/accuracy_2_of_5‚1 7 Purity/*FRt*SRqqt*FBt/FRt_purity‚0.1574157476425171 9"Purity/*FRt0SRqqt1FBt/event_purity‚0.8596229553222656 ,&Purity/0FRt2SRqqt2FBt/event_proportion‚0 6 Purity/1FRt*SRqqt1FBt/FRt_purity‚0.532058596611023 7 Purity/1FRt1SRqqt2FBt/FBt_purity‚0.7222222089767456 ,&Purity/2FRt2SRqqt2FBt/event_proportion‚0 /particle/accuracy_4_of_4‚0.9418045878410339 7 Purity/*FRt*SRqqt1FBt/FBt_purity‚0.9923688173294067 7 Purity/*FRt1SRqqt2FBt/FBt_purity‚0.8194444179534912 =&Purity/0FRt0SRqqt0FBt/event_proportion‚0.0462472103536129 9"Purity/0FRt1SRqqt*FBt/SRqqt_purity‚0.5799221992492676 7 Purity/1FRt0SRqqt0FBt/FRt_purity‚0.1532375067472458 8 Purity/2FRt*SRqqt0FBt/FRt_purity‚0.15984416007995605 /particle/accuracy_3_of_4‚0.9946429133415222 'particle/f_score‚0.6116828322410583 +"Purity/*FRt*SRqqt2FBt/SRqqt_purity‚0.25 9"Purity/1FRt*SRqqt*FBt/SRqqt_purity‚0.6988366842269897 =&Purity/1FRt0SRqqt0FBt/event_proportion‚0.4408714771270752 8&Purity/1FRt2SRqqt0FBt/event_proportion‚ 0.02294921875 8 Purity/1FRt2SRqqt1FBt/FRt_purity‚0.10000000149011612 particle/accuracy_1_of_1‚1 9"Purity/0FRt*SRqqt0FBt/SRqqt_purity‚0.5754846334457397 =&Purity/2FRt*SRqqt0FBt/event_proportion‚0.0382486991584301 8"Purity/2FRt2SRqqt0FBt/SRqqt_purity‚0.759257972240448 +particle/sensitivity‚0.6023409366607666 =&Purity/*FRt0SRqqt1FBt/event_proportion‚0.0418526791036129 =&Purity/0FRt0SRqqt1FBt/event_proportion‚0.0313895083963871 9"Purity/0FRt1SRqqt1FBt/event_purity‚0.4493831396102905 9"Purity/1FRt1SRqqt*FBt/SRqqt_purity‚0.7087523341178894 7 Purity/1FRt1SRqqt0FBt/FRt_purity‚0.1157868281006813 ("Purity/1FRt1SRqqt2FBt/event_purity‚0 ("Purity/2FRt2SRqqt*FBt/event_purity‚0 epoch‚6 8 Purity/*FRt1SRqqt0FBt/FRt_purity‚0.13207942247390747 7 Purity/0FRt1SRqqt1FBt/FBt_purity‚0.9846875071525574 7 Purity/1FRt*SRqqt2FBt/FRt_purity‚0.8463152050971985 7 Purity/1FRt0SRqqt2FBt/FBt_purity‚0.9970238208770752 & Purity/2FRt*SRqqt2FBt/FBt_purity‚1 jet/accuracy_3_of_5‚1 9"Purity/*FRt0SRqqt2FBt/event_purity‚0.8548386693000793 9"Purity/0FRt*SRqqt1FBt/event_purity‚0.8290182948112488 9"Purity/0FRt0SRqqt1FBt/event_purity‚0.9922443628311157 ("Purity/2FRt1SRqqt0FBt/event_purity‚0 9"Purity/0FRt2SRqqt0FBt/SRqqt_purity‚0.5441195368766785 6 Purity/1FRt0SRqqt*FBt/FBt_purity‚0.996457576751709 ("Purity/1FRt2SRqqt1FBt/event_purity‚0 & Purity/2FRt*SRqqt*FBt/FBt_purity‚1 particle/accuracy_1_of_5‚1 =&Purity/0FRt1SRqqt*FBt/event_proportion‚0.2203543484210968 9"Purity/1FRt0SRqqt0FBt/event_purity‚0.1532375067472458 ("Purity/2FRt1SRqqt*FBt/event_purity‚0 7 Purity/2FRt0SRqqt1FBt/FRt_purity‚0.5260416865348816 9"Purity/*FRt0SRqqt*FBt/event_purity‚0.2195509672164917 9"Purity/*FRt2SRqqt0FBt/SRqqt_purity‚0.6227667331695557 8 Purity/2FRt2SRqqt0FBt/FRt_purity‚0.08588450402021408 7 Purity/0FRt*SRqqt1FBt/FBt_purity‚0.9904049634933472 9"Purity/0FRt0SRqqt*FBt/event_purity‚0.9803967475891113 @&Purity/1FRt0SRqqt2FBt/event_proportion‚0.0024879092816263437 6 Purity/1FRt1SRqqt*FBt/FBt_purity‚0.980157196521759 & Purity/2FRt1SRqqt1FBt/FBt_purity‚1 +jet/accuracy_1_of_1‚0.28393423557281494 9"Purity/*FRt*SRqqt1FBt/event_purity‚0.6860703825950623 7 Purity/*FRt1SRqqt*FBt/FBt_purity‚0.9792560338973999 8 Purity/*FRt2SRqqt1FBt/FRt_purity‚0.10000000149011612 =&Purity/1FRt0SRqqt*FBt/event_proportion‚0.4530552327632904 8 Purity/2FRt1SRqqt*FBt/FRt_purity‚0.24550892412662506 ,&Purity/2FRt2SRqqt1FBt/event_proportion‚0 jet/accuracy_4_of_5‚0 :"Purity/*FRt0SRqqt0FBt/event_purity‚0.14732594788074493 =&Purity/*FRt0SRqqt2FBt/event_proportion‚0.0103004090487957 *jet/accuracy_2_of_4‚0.7170566916465759 =&Purity/*FRt0SRqqt0FBt/event_proportion‚0.4607747495174408 9"Purity/*FRt1SRqqt*FBt/event_purity‚0.3433382213115692 9"Purity/1FRt*SRqqt2FBt/event_purity‚0.6288359761238098 @&Purity/2FRt1SRqqt1FBt/event_proportion‚0.0008370535797439516 particle/accuracy_1_of_3‚1 7 Purity/2FRt0SRqqt2FBt/FRt_purity‚0.6428571343421936 9"Purity/*FRt*SRqqt2FBt/event_purity‚0.7989703416824341 @&Purity/0FRt1SRqqt2FBt/event_proportion‚0.0004417782765813172 & Purity/1FRt2SRqqt*FBt/FBt_purity‚1 8"Purity/2FRt1SRqqt*FBt/SRqqt_purity‚0.712454617023468 9"Purity/2FRt0SRqqt2FBt/event_purity‚0.4285714328289032 & Purity/2FRt0SRqqt2FBt/FBt_purity‚1 _runtime‚1941.192762212 <&Purity/*FRt*SRqqt0FBt/event_proportion‚0.881045401096344 ?&Purity/0FRt0SRqqt2FBt/event_proportion‚0.007626487873494625 >&Purity/2FRt0SRqqt0FBt/event_proportion‚0.01990327425301075 *jet/accuracy_2_of_3‚0.5014821887016296 % _timestamp‚1.7844622668036954e+09 =&Purity/*FRt1SRqqt0FBt/event_proportion‚0.3670479953289032 9"Purity/0FRt0SRqqt2FBt/event_purity‚0.9263659119606018 9"Purity/1FRt1SRqqt2FBt/SRqqt_purity‚0.3333333432674408 particle/accuracy_1_of_4‚1 particle/accuracy_5_of_5‚0 8 Purity/*FRt2SRqqt0FBt/FRt_purity‚0.09270299971103668 @&Purity/0FRt2SRqqt1FBt/event_proportion‚0.0007207961170934141 :"Purity/1FRt1SRqqt1FBt/event_purity‚0.20270052552223206 9"Purity/2FRt1SRqqt1FBt/SRqqt_purity‚0.7129629254341125 =&Purity/0FRt*SRqqt*FBt/event_proportion‚0.2839471697807312 9"Purity/0FRt1SRqqt0FBt/SRqqt_purity‚0.5831332206726074 8 Purity/2FRt0SRqqt0FBt/FRt_purity‚0.13748490810394287 @&Purity/2FRt0SRqqt1FBt/event_proportion‚0.0007672990905120969 9"Purity/*FRt1SRqqt*FBt/SRqqt_purity‚0.6353863477706909 9"Purity/1FRt0SRqqt2FBt/event_purity‚0.6821807026863098 @&Purity/1FRt1SRqqt2FBt/event_proportion‚0.0002325148816453293 ("Purity/2FRt1SRqqt2FBt/SRqqt_purity‚0 @&Purity/*FRt1SRqqt2FBt/event_proportion‚0.0006975446594879031 & Purity/0FRt2SRqqt*FBt/FBt_purity‚1 7 Purity/1FRt*SRqqt*FBt/FBt_purity‚0.9905831217765808 9"Purity/1FRt1SRqqt0FBt/SRqqt_purity‚0.7133118510246277 9"Purity/1FRt2SRqqt0FBt/SRqqt_purity‚0.6695426106452942 +jet/accuracy_2_of_2‚0.12091930955648422 ;"Purity/2FRt0SRqqt0FBt/event_purity‚0.015115242451429367 7 Purity/1FRt*SRqqt2FBt/FBt_purity‚0.9767574071884155 7 Purity/1FRt0SRqqt2FBt/FRt_purity‚0.8551587462425232–ñ­Ø÷õ Ø{ 9"Purity/*FRt*SRqqt1FBt/event_purity‚0.6860703825950623 7 Purity/*FRt1SRqqt*FBt/FBt_purity‚0.9792560338973999 8 Purity/*FRt2SRqqt1FBt/FRt_purity‚0.10000000149011612 =&Purity/1FRt0SRqqt*FBt/event_proportion‚0.4530552327632904 8 Purity/2FRt1SRqqt*FBt/FRt_purity‚0.24550892412662506 ,&Purity/2FRt2SRqqt1FBt/event_proportion‚0 jet/accuracy_4_of_5‚0 :"Purity/*FRt0SRqqt0FBt/event_purity‚0.14732594788074493 =&Purity/*FRt0SRqqt2FBt/event_proportion‚0.0103004090487957 *jet/accuracy_2_of_4‚0.7170566916465759 =&Purity/*FRt0SRqqt0FBt/event_proportion‚0.4607747495174408 9"Purity/*FRt1SRqqt*FBt/event_purity‚0.3433382213115692 9"Purity/1FRt*SRqqt2FBt/event_purity‚0.6288359761238098 @&Purity/2FRt1SRqqt1FBt/event_proportion‚0.0008370535797439516 particle/accuracy_1_of_3‚1 7 Purity/2FRt0SRqqt2FBt/FRt_purity‚0.6428571343421936 9"Purity/*FRt*SRqqt2FBt/event_purity‚0.7989703416824341 @&Purity/0FRt1SRqqt2FBt/event_proportion‚0.0004417782765813172 & Purity/1FRt2SRqqt*FBt/FBt_purity‚1 8"Purity/2FRt1SRqqt*FBt/SRqqt_purity‚0.712454617023468 9"Purity/2FRt0SRqqt2FBt/event_purity‚0.4285714328289032 & Purity/2FRt0SRqqt2FBt/FBt_purity‚1 _runtime‚1941.192762212 <&Purity/*FRt*SRqqt0FBt/event_proportion‚0.881045401096344 ?&Purity/0FRt0SRqqt2FBt/event_proportion‚0.007626487873494625 >&Purity/2FRt0SRqqt0FBt/event_proportion‚0.01990327425301075 *jet/accuracy_2_of_3‚0.5014821887016296 % _timestamp‚1.7844622668036954e+09 =&Purity/*FRt1SRqqt0FBt/event_proportion‚0.3670479953289032 9"Purity/0FRt0SRqqt2FBt/event_purity‚0.9263659119606018 9"Purity/1FRt1SRqqt2FBt/SRqqt_purity‚0.3333333432674408 particle/accuracy_1_of_4‚1 particle/accuracy_5_of_5‚0 8 Purity/*FRt2SRqqt0FBt/FRt_purity‚0.09270299971103668 @&Purity/0FRt2SRqqt1FBt/event_proportion‚0.0007207961170934141 :"Purity/1FRt1SRqqt1FBt/event_purity‚0.20270052552223206 9"Purity/2FRt1SRqqt1FBt/SRqqt_purity‚0.7129629254341125 =&Purity/0FRt*SRqqt*FBt/event_proportion‚0.2839471697807312 9"Purity/0FRt1SRqqt0FBt/SRqqt_purity‚0.5831332206726074 8 Purity/2FRt0SRqqt0FBt/FRt_purity‚0.13748490810394287 @&Purity/2FRt0SRqqt1FBt/event_proportion‚0.0007672990905120969 9"Purity/*FRt1SRqqt*FBt/SRqqt_purity‚0.6353863477706909 9"Purity/1FRt0SRqqt2FBt/event_purity‚0.6821807026863098 @&Purity/1FRt1SRqqt2FBt/event_proportion‚0.0002325148816453293 ("Purity/2FRt1SRqqt2FBt/SRqqt_purity‚0 @&Purity/*FRt1SRqqt2FBt/event_proportion‚0.0006975446594879031 & Purity/0FRt2SRqqt*FBt/FBt_purity‚1 7 Purity/1FRt*SRqqt*FBt/FBt_purity‚0.9905831217765808 9"Purity/1FRt1SRqqt0FBt/SRqqt_purity‚0.7133118510246277 9"Purity/1FRt2SRqqt0FBt/SRqqt_purity‚0.6695426106452942 +jet/accuracy_2_of_2‚0.12091930955648422 ;"Purity/2FRt0SRqqt0FBt/event_purity‚0.015115242451429367 7 Purity/1FRt*SRqqt2FBt/FBt_purity‚0.9767574071884155 7 Purity/1FRt0SRqqt2FBt/FRt_purity‚0.8551587462425232 7 Purity/1FRt1SRqqt1FBt/FRt_purity‚0.4525213837623596 * Purity/2FRt*SRqqt2FBt/FRt_purity‚0.625 7 Purity/*FRt1SRqqt1FBt/FRt_purity‚0.4562568664550781 9"Purity/*FRt1SRqqt1FBt/SRqqt_purity‚0.5725939869880676 & Purity/*FRt2SRqqt*FBt/FBt_purity‚1 8"Purity/0FRt1SRqqt*FBt/event_purity‚0.574561357498169 9"Purity/1FRt*SRqqt0FBt/event_purity‚0.1184980720281601 A&Purity/2FRt0SRqqt2FBt/event_proportion‚0.00018601190822664648 +jet/accuracy_3_of_4‚0.06511863321065903 trainer/global_step‚2799 9"Purity/*FRt*SRqqt*FBt/event_purity‚0.2648735046386719 @&Purity/*FRt2SRqqt1FBt/event_proportion‚0.0009998140158131719 7 Purity/0FRt0SRqqt1FBt/FBt_purity‚0.9922443628311157 =&Purity/0FRt1SRqqt0FBt/event_proportion‚0.2083565890789032 9"Purity/0FRt2SRqqt*FBt/SRqqt_purity‚0.5392657518386841 8 Purity/1FRt2SRqqt0FBt/FRt_purity‚0.09434112161397934 <&Purity/*FRt2SRqqt*FBt/event_proportion‚0.054222472012043 ("Purity/0FRt2SRqqt1FBt/event_purity‚0 9"Purity/1FRt*SRqqt2FBt/SRqqt_purity‚0.3333333432674408 :"Purity/2FRt*SRqqt1FBt/event_purity‚0.07000000029802322 jet/accuracy_1_of_5‚1 9"Purity/*FRt*SRqqt0FBt/SRqqt_purity‚0.6356189250946045 =&Purity/*FRt*SRqqt1FBt/event_proportion‚0.0617094486951828 ) Purity/*FRt1SRqqt2FBt/FRt_purity‚0.75 7 Purity/1FRt*SRqqt0FBt/FRt_purity‚0.1419883817434311 :"Purity/1FRt1SRqqt*FBt/event_purity‚0.03964243829250336 8"Purity/2FRt2SRqqt*FBt/SRqqt_purity‚0.759257972240448 /particle/accuracy_3_of_3‚0.9994933605194092 7 Purity/1FRt0SRqqt1FBt/FRt_purity‚0.5917165279388428 7 Purity/1FRt1SRqqt2FBt/FRt_purity‚0.7777777910232544 7 Pº‰dç+urity/2FRt*SRqqt1FBt/FRt_purity‚0.4820833206176758 7 Purity/*FRt0SRqqt*FBt/FRt_purity‚0.1657065600156784 :"Purity/*FRt1SRqqt0FBt/event_purity‚0.34413379430770874 :"Purity/1FRt*SRqqt*FBt/event_purity‚0.12698841094970703 A&Purity/1FRt2SRqqt1FBt/event_proportion‚0.00027901786961592734 9"Purity/2FRt*SRqqt0FBt/SRqqt_purity‚0.7366169691085815 ?&Purity/2FRt2SRqqt*FBt/event_proportion‚0.006417410913854837 & Purity/2FRt0SRqqt*FBt/FBt_purity‚1 7 Purity/*FRt0SRqqt1FBt/FBt_purity‚0.9935435056686401 :"Purity/*FRt2SRqqt1FBt/SRqqt_purity‚0.35648149251937866 =&Purity/*FRt1SRqqt1FBt/event_proportion‚0.0188569575548172 7 Purity/0FRt0SRqqt2FBt/FBt_purity‚0.9631829261779785 =&Purity/2FRt0SRqqt*FBt/event_proportion‚0.0208565853536129 @&Purity/2FRt1SRqqt2FBt/event_proportion‚2.325148852833081e-05 8 Purity/*FRt0SRqqt0FBt/FRt_purity‚0.15204337239265442 ?&Purity/0FRt1SRqqt1FBt/event_proportion‚0.011555989272892475 =&Purity/1FRt*SRqqt0FBt/event_proportion‚0.6105840802192688 9"Purity/1FRt*SRqqt0FBt/SRqqt_purity‚0.7030450105667114 ,&Purity/1FRt2SRqqt2FBt/event_proportion‚0 =&Purity/0FRt*SRqqt1FBt/event_proportion‚0.0436662957072258 & Purity/0FRt2SRqqt1FBt/FBt_purity‚1 >&Purity/1FRt*SRqqt1FBt/event_proportion‚0.01643880270421505 7 Purity/1FRt*SRqqt1FBt/FBt_purity‚0.9967347383499146 ;"Purity/2FRt0SRqqt*FBt/event_purity‚0.025742391124367714 particle/accuracy_2_of_2‚1 7 Purity/*FRt*SRqqt1FBt/FRt_purity‚0.5231402516365051 ("Purity/*FRt2SRqqt1FBt/event_purity‚0 7 Purity/1FRt*SRqqt*FBt/FRt_purity‚0.1552247703075409 :"Purity/*FRt*SRqqt0FBt/event_purity‚0.22861839830875397 +"Purity/*FRt1SRqqt2FBt/SRqqt_purity‚0.25 9"Purity/0FRt*SRqqt*FBt/SRqqt_purity‚0.5721141695976257 & Purity/1FRt2SRqqt1FBt/FBt_purity‚1 ;"Purity/2FRt*SRqqt*FBt/event_purity‚0.013760234229266644 @&Purity/2FRt*SRqqt1FBt/event_proportion‚0.0016043527284637094 +jet/accuracy_3_of_3‚0.05433806777000427 7 Purity/*FRt0SRqqt2FBt/FBt_purity‚0.9719924926757812 :"Purity/*FRt1SRqqt1FBt/event_purity‚0.34146004915237427 @&Purity/1FRt*SRqqt2FBt/event_proportion‚0.0027204242069274187 >&Purity/1FRt2SRqqt*FBt/event_proportion‚0.02322823740541935 7 Purity/1FRt2SRqqt*FBt/FRt_purity‚0.0938316211104393 *loss/val_total_loss‚12.278507548993472 7 Purity/*FRt*SRqqt2FBt/FRt_purity‚0.8267573118209839 9"Purity/*FRt1SRqqt0FBt/SRqqt_purity‚0.6393875479698181 ?&Purity/0FRt*SRqqt2FBt/event_proportion‚0.008068266324698925 ;"Purity/2FRt*SRqqt0FBt/event_purity‚0.008214597590267658 6validation_average_jet_accuracy‚0.3298458456993103 :"Purity/0FRt*SRqqt2FBt/SRqqt_purity‚0.19230769574642181 /"Purity/0FRt2SRqqt1FBt/SRqqt_purity‚0.390625 =&Purity/1FRt1SRqqt0FBt/event_proportion‚0.1467633992433548 9"Purity/2FRt*SRqqt1FBt/SRqqt_purity‚0.7129629254341125 8&Purity/*FRt2SRqqt0FBt/event_proportion‚ 0.05322265625 >&Purity/0FRt2SRqqt*FBt/event_proportion‚0.02457682229578495 9"Purity/*FRt2SRqqt*FBt/SRqqt_purity‚0.6179335713386536 *jet/accuracy_1_of_3‚0.8990447521209717 ,"Purity/2FRt*SRqqt2FBt/event_purity‚0.375 ( Purity/2FRt1SRqqt2FBt/FRt_purity‚0.5 ,jet/accuracy_4_of_4‚0.013899499550461769 & Purity/2FRt0SRqqt1FBt/FBt_purity‚1 7 Purity/*FRt0SRqqt*FBt/FBt_purity‚0.9869740009307861 8 Purity/2FRt0SRqqt*FBt/FRt_purity‚0.15635846555233002 =&Purity/*FRt0SRqqt*FBt/event_proportion‚0.5129278302192688 8 Purity/*FRt2SRqqt*FBt/FRt_purity‚0.09250184893608093 9"Purity/0FRt1SRqqt0FBt/event_purity‚0.5831332206726074 ("Purity/1FRt2SRqqt*FBt/event_purity‚0 7 Purity/2FRt1SRqqt1FBt/FRt_purity‚0.4513888955116272 & Purity/*FRt2SRqqt1FBt/FBt_purity‚1 @&Purity/1FRt1SRqqt1FBt/event_proportion‚0.0064639137126505375 :"Purity/0FRt1SRqqt2FBt/SRqqt_purity‚0.19230769574642181 particle/accuracy_4_of_5‚1 7 Purity/0FRt0SRqqt*FBt/FBt_purity‚0.9836359620094299 ;"Purity/1FRt1SRqqt0FBt/event_purity‚0.032493408769369125 ("Purity/1FRt2SRqqt0FBt/event_purity‚0 9"Purity/1FRt1SRqqt1FBt/SRqqt_purity‚0.6245261430740356 ("Purity/2FRt1SRqqt1FBt/event_purity‚0 ("Purity/*FRt1SRqqt2FBt/event_purity‚0 :"Purity/1FRt0SRqqt*FBt/event_purity‚0.16305947303771973 :"Purity/1FRt0SRqqt1FBt/event_purity‚0.48606768250465393 9"Purity/2FRt1SRqqt0FBt/SRqqt_purity‚0.7159510254859924 9"Purity/*FRt*SRqqt1FBt/SRqqt_purity‚0.5510120987892151 8"Purity/0FRt2SRqqt0FBt/event_purity‚0.309317022562027 ?&Purity/2FRt2SRqqt0FBt/event_proportion‚0.006417410913854837 particle/accuracy_2_of_4‚1 7"Purity/0FRt*SRqqt2FBt/event_purity‚0.87195885181427 ("Purity/0FRt0SRqqt0FBt/event_purity‚1 8 Purity/2FRt1SRqqt0FBt/FRt_purity‚0.23232655227184296 jet/accuracy_5_of_5‚0 :"Purity/*FRt2SRqqt*FBt/event_purity‚0.13411442935466766 :"Purity/*FRt2SRqqt0FBt/event_purity‚0.13668189942836761 7 Purity/0FRt1SRqqt*FBt/FBt_purity‚0.9789730906486511 =&Purity/1FRt*SRqqt*FBt/event_proportion‚0.6297432780265808 =&Purity/2FRt1SRqqt*FBt/event_proportion‚0.0127883180975914 )jet/accuracy_1_of_2‚0.752110481262207 9"Purity/0FRt*SRqqt1FBt/SRqqt_purity‚0.5201519727706909 8 Purity/2FRt2SRqqt*FBt/FRt_purity‚0.08588450402021408 :"Purity/1FRt2SRqqt1FBt/SRqqt_purity‚0.32499998807907104 ?&Purity/2FRt1SRqqt0FBt/event_proportion‚0.011928013525903225 *jet/accuracy_1_of_4‚0.9432007670402527 9"Purity/2FRt0SRqqt1FBt/event_purity‚0.1822916716337204 _step‚230 6 Purity/0FRt*SRqqt2FBt/FBt_purity‚0.958831787109375 >&Purity/0FRt2SRqqt0FBt/event_proportion‚0.02385602705180645 9"Purity/1FRt2SRqqt*FBt/SRqqt_purity‚0.6652831435203552 & Purity/2FRt1SRqqt2FBt/FBt_purity‚1 7 Purity/*FRt*SRqqt*FBt/FBt_purity‚0.9852825403213501 +particle/specificity‚0.8996248841285706 8 Purity/*FRt1SRqqt*FBt/FRt_purity‚0.14784999191761017 6 Purity/0FRt*SRqqt*FBt/FBt_purity‚0.982567310333252 7 Purity/1FRt1SRqqt1FBt/FBt_purity‚0.9965986609458923 & Purity/2FRt1SRqqt*FBt/FBt_purity‚1 7 Purity/*FRt*SRqqt2FBt/FBt_purity‚0.9641268849372864 =&Purity/*FRt1SRqqt*FBt/event_proportion‚0.3866024911403656 7 Purity/*FRt1SRqqt1FBt/FBt_purity‚0.9889274835586548 =&Purity/0FRt0SRqqt*FBt/event_proportion‚0.0390159972012043 6 Purity/1FRt1SRqqt*FBt/FRt_purity‚0.131504088640213 particle/accuracy_2_of_3‚1 ("Purity/0FRt1SRqqt2FBt/event_purity‚0 7 Purity/*FRt0SRqqt2FBt/FRt_purity‚0.8370748162269592 9"Purity/0FRt1SRqqt1FBt/SRqqt_purity‚0.5403477549552917 =&Purity/1FRt1SRqqt*FBt/event_proportion‚0.1534598171710968 @&Purity/2FRt*SRqqt2FBt/event_proportion‚0.0002092633949359879 8 Purity/1FRt0SRqqt*FBt/FRt_purity‚0.16644589602947235 8 Purity/2FRt*SRqqt*FBt/FRt_purity‚0.17520101368427277 & Purity/2FRt*SRqqt1FBt/FBt_purity‚1 8 Purity/*FRt*SRqqt0FBt/FRt_purity‚0.14393380284309387 ,&Purity/*FRt2SRqqt2FBt/event_proportion‚0 9"Purity/0FRt*SRqqt0FBt/event_purity‚0.5545304417610168 9"Purity/1FRt*SRqqt1FBt/SRqqt_purity‚0.6057761311531067 7 Purity/1FRt0SRqqt1FBt/FBt_purity‚0.9960317015647888 ("Purity/2FRt2SRqqt0FBt/event_purity‚0 (particle/accuracy‚0.8357862830162048 9"Purity/0FRt*SRqqt*FBt/event_purity‚0.6061073541641235 >&Purity/1FRt0SRqqt1FBt/event_proportion‚0.00969587080180645 9"Purity/2FRt*SRqqt*FBt/SRqqt_purity‚0.7340572476387024 particle/accuracy_1_of_2‚1 7 Purity/0FRt1SRqqt2FBt/FBt_purity‚0.9038461446762085 ("Purity/2FRt*SRqqt2FBt/SRqqt_purity‚0 particle/accuracy_3_of_5‚1 =&Purity/*FRt*SRqqt*FBt/event_proportion‚0.9537528157234192 ?&Purity/*FRt*SRqqt2FBt/event_proportion‚0.010997953824698925 7 Purity/*FRt0SRqqt1FBt/FRt_purity‚0.5799595713615417 =&Purity/0FRt*SRqqt0FBt/event_proportion‚0.2322126179933548 <&Purity/2FRt*SRqqt*FBt/event_proportion‚0.040062315762043 9"Purity/*FRt*SRqqt*FBt/SRqqt_purity‚0.6315377354621887 :"Purity/0FRt2SRqqt*FBt/event_purity‚0.30029502511024475 :"Purity/1FRt*SRqqt1FBt/event_purity‚0.36711403727531433 ("Purity/2FRt1SRqqt2FBt/event_purity‚0 jet/accuracy_2_of_5‚1 particle/accuracy_2_of_5‚1 7 Purity/*FRt*SRqqt*FBt/FRt_purity‚0.1574157476425171 9"Purity/*FRt0SRqqt1FBt/event_purity‚0.8596229553222656 ,&Purity/0FRt2SRqqt2FBt/event_proportion‚0 6 Purity/1FRt*SRqqt1FBt/FRt_purity‚0.532058596611023 7 Purity/1FRt1SRqqt2FBt/FBt_purity‚0.7222222089767456 ,&Purity/2FRt2SRqqt2FBt/event_proportion‚0 /particle/accuracy_4_of_4‚0.9418045878410339 7 Purity/*FRt*SRqqt1FBt/FBt_purity‚0.9923688173294067 7 Purity/*FRt1SRqqt2FBt/FBt_purity‚0.8194444179534912 =&Purity/0FRt0SRqqt0FBt/event_proportion‚0.0462472103536129 9"Purity/0FRt1SRqqt*FBt/SRqqt_purity‚0.5799221992492676 7 Purity/1FRt0SRqqt0FBt/FRt_purity‚0.1532375067472458 8 Purity/2FRt*SRqqt0FBt/FRt_purity‚0.15984416007995605 /particle/accuracy_3_of_4‚0.9946429133415222 'particle/f_score‚0.6116828322410583 +"Purity/*FRt*SRqqt2FBt/SRqqt_purity‚0.25 9"Purity/1FRt*SRqqt*FBt/SRqqt_purity‚0.6988366842269897 =&Purity/1FRt0SRqqt0FBt/event_proportion‚0.4408714771270752 8&Purity/1FRt2SRqqt0FBt/event_proportion‚ 0.02294921875 8 Purity/1FRt2SRqqt1FBt/FRt_purity‚0.10000000149011612 particle/accuracy_1_of_1‚1 9"Purity/0FRt*SRqqt0FBt/SRqqt_purity‚0.5754846334457397 =&Purity/2FRt*SRqqt0FBt/event_proportion‚0.0382486991584301 8"Purity/2FRt2SRqqt0FBt/SRqqt_purity‚0.759257972240448 +particle/sensitivity‚0.6023409366607666 =&Purity/*FRt0SRqqt1FBt/event_proportion‚0.0418526791036129 =&Purity/0FRt0SRqqt1FBt/event_proportion‚0.0313895083963871 9"Purity/0FRt1SRqqt1FBt/event_purity‚0.4493831396102905 9"Purity/1FRt1SRqqt*FBt/SRqqt_purity‚0.7087523341178894 7 Purity/1FRt1SRqqt0FBt/FRt_purity‚0.1157868281006813 ("Purity/1FRt1SRqqt2FBt/event_purity‚0 ("Purity/2FRt2SRqqt*FBt/event_purity‚0 epoch‚6 8 Purity/*FRt1SRqqt0FBt/FRt_purity‚0.13207942247390747 7 Purity/0FRt1SRqqt1FBt/FBt_purity‚0.9846875071525574 7 Purity/1FRt*SRqqt2FBt/FRt_purity‚0.8463152050971985 7 Purity/1FRt0SRqqt2FBt/FBt_purity‚0.9970238208770752 & Purity/2FRt*SRqqt2FBt/FBt_purity‚1 jet/accuracy_3_of_5‚1 9"Purity/*FRt0SRqqt2FBt/event_purity‚0.8548386693000793 9"Purity/0FRt*SRqqt1FBt/event_purity‚0.8290182948112488 9"Purity/0FRt0SRqqt1FBt/event_purity‚0.9922443628311157 ("Purity/2FRt1SRqqt0FBt/event_purity‚0 9"Purity/0FRt2SRqqt0FBt/SRqqt_purity‚0.5441195368766785 6 Purity/1FRt0SRqqt*FBt/FBt_purity‚0.996457576751709 ("Purity/1FRt2SRqqt1FBt/event_purity‚0 & Purity/2FRt*SRqqt*FBt/FBt_purity‚1 particle/accuracy_1_of_5‚1 =&Purity/0FRt1SRqqt*FBt/event_proportion‚0.2203543484210968 9"Purity/1FRt0SRqqt0FBt/event_purity‚0.1532375067472458 ("Purity/2FRt1SRqqt*FBt/event_purity‚0 7 Purity/2FRt0SRqqt1FBt/FRt_purity‚0.5260416865348816 9"Purity/*FRt0SRqqt*FBt/event_purity‚0.2195509672164917 9"Purity/*FRt2SRqqt0FBt/SRqqt_purity‚0.6227667331695557 8 Purity/2FRt2SRqqt0FBt/FRt_purity‚0.08588450402021408 7 Purity/0FRt*SRqqt1FBt/FBt_purity‚0.9904049634933472 9"Purity/0FRt0SRqqt*FBt/event_purity‚0.9803967475891113 @&Purity/1FRt0SRqqt2FBt/event_proportion‚0.0024879092816263437 6 Purity/1FRt1SRqqt*FBt/FBt_purity‚0.980157196521759 & Purity/2FRt1SRqqt1FBt/FBt_purity‚1 +jet/accuracy_1_of_1‚0.28393423557281494æî´¤Ãaö ‚(:Ö Ã÷òÒ³±˜¥ disk.md0.in‚0! disk.dm-1.out‚1387.8056640625 disk.rbd0.out‚0) proc.memory.percent‚18.09544563293457 network.sent‚ 4.126038e+06 network.recv‚539810 disk.dm-0.in‚ 1982.2109375! disk.dm-0.out‚38002.642578125 disk.dm-1.in‚0 disk.nvme0c0n1.out‚24790.75 disk.nvme0n1.in‚ 1852.4453125 disk.nvme0n1.out‚24790.75% disk./.usageGB‚1171.1341285705566 disk.md0.out‚ 49.32421875 disk.nvme1c1n1.in‚ 129.765625 disk.nvme1n1.in‚ 129.765625% disk.nvme1n1.out‚14705.7255859375" proc.memory.rssMB‚ 2964.7578125 proc.cpu.threads‚17 disk.rbd0.in‚0" disk.nvme0c0n1.in‚ 1852.4453125' disk.nvme1c1n1.out‚14705.7255859375$ memory_percent‚98.81308078765869( proc.memory.availableMB‚ 194.46484375 cpu‚25.85710146439656) disk./.usagePercent‚76.28295903406979Wã¨÷ ‚(:’ Ã÷òÒ‘‚¦ gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚48.0 gpu.0.powerWatts‚118.102' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚39.367333333333335 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Oa¾‚nø ‚(:ã Ò÷òÒ˜…Ĥ# disk.nvme1n1.out‚14771.51171875 disk.dm-0.in‚ 1982.2109375# disk.dm-0.out‚38022.99658203125 disk.dm-1.in‚0" disk.nvme0c0n1.in‚ 1852.4453125 disk.nvme0n1.in‚ 1852.4453125 disk.md0.in‚0 disk.md0.out‚ 49.66796875& disk.nvme0n1.out‚24810.02880859375 disk.nvme1c1n1.in‚ 129.765625% disk.nvme1c1n1.out‚14771.51171875 proc.cpu.threads‚17 network.recv‚544337 disk.rbd0.out‚0# memory_percent‚98.8133430480957 cpu‚25.85554895180295% disk./.usageGB‚1171.1341361999512 disk.nvme1n1.in‚ 129.765625! disk.dm-1.out‚1451.7763671875 disk.rbd0.in‚0( disk.nvme0c0n1.out‚24810.02880859375& proc.memory.availableMB‚ 194.421875" proc.memory.rssMB‚ 2964.7578125) proc.memory.percent‚18.09544563293457 network.sent‚ 4.183906e+06) disk./.usagePercent‚76.28295953101782ì¾WÅžù ‚(:“ Ò÷òÒæØû¤ gpu.0.gpu‚91.0 gpu.0.memory‚17, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚48.0 gpu.0.powerWatts‚225.409' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚75.13633333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ÈâE¹2ú - _wandbruntime‚1973  _runtime‚1973‚Êö Éû à _runtime‚1973.484284913 _step‚231 ' lr-AdamW/pg1‚0.00013394725470973996 ' lr-AdamW/pg2‚0.00013394725470973996 trainer/global_step‚2849 % _timestamp‚1.7844622990959184e+09Ò²Îü È trainer/global_step‚2849 % _timestamp‚1.7844622990959184e+09 _runtime‚1973.484284913 _step‚231 ' lr-AdamW/pg1‚0.00013394725470973996 ' lr-AdamW/pg2‚0.00013394725470973996ç3#/£2ý - _wandbruntime‚1973  _runtime‚1973Ð#¹à'þ ÚO E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚15168686 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚6463269 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚220905956300204 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚182 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚165661339325952 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 trainer/global_step‚2849 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚218560704280816 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 213067776 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2421762575360 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 _step‚232 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚22599223 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚220905852474418 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚223522848810496 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚6463091 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚221171129235968 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 118958080 % _timestamp‚1.7844622990968838e+09 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚223522743111168 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2421643617280 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚28677387 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2351613875200 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚28677387 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚218560687241456 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚51266465 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2351702535168 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚168083196011008 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚22599221 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚223522743111168 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚221171129235968 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚21631955 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚21631773 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2345165232962 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2345252019388 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2351613875200 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 _runtime‚1973.485111514 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2345252019388 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2351613875200 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 _runtime‚1973.485111514 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 213067776 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2421762575360 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 _step‚232 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚22599223 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚220905852474418 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚223522848810496 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚6463091 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚221171129235968 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 118958080 % _timestamp‚1.7844622990968838e+09 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 JBDeviceStatsMonitor.on_train_batch_start/segment.largüŠÜþe_pool.current‚682 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚223522743111168 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2421643617280 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚28677387 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2351613875200 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚28677387 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚218560687241456 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚51266465 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2351702535168 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚168083196011008 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚22599221 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚223522743111168 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚221171129235968 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚21631955 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚21631773 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2345165232962èTH«2€ - _wandbruntime‚1973  _runtime‚1973œ™Óûê& äM H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚28677064 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2352399590912 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 223546368 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚22606850 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚51294073 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚15173738 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚165717571163648 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 _runtime‚1973.892829987 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚22606850 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚218634977766848 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2346035514100 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚21638983 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2352399590912 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚22606852 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚187 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚168140148708352 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚165717477053952 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 % _timestamp‚1.7844622995042343e+09 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6465245 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚51283914 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚28677064 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚191 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚220980926494410 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚28687221 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 trainer/global_step‚2849 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚223598784199680 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚21639174 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚168139925161984 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2422577544704 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚218634994806208 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚51294073 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚22606852 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚51283914 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚28687221 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2422448108032 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚221246278902272 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚220981030320308 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚221246295941632 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 129436672 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682>ªVï&‚ éM [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚221246295941632 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 129436672 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚28677064 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2352399590912 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 223546368 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚22606850 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚51294073 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚15173738 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚165717571163648 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 _runtime‚1973.892829987 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚22606850 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚218634977766848 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2346035514100 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚21638983 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2352399590912 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚22606852 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚187 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚168140148708352 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚165717477053952 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 % _timestamp‚1.7844622995042343e+09 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6465245 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚51283914 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚28677064 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚191 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚220980926494410 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚28687221 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 trainer/global_step‚2849 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚223598784199680 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚21639174 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚168139925161984 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2422577544704 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚218634994806208 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚51294073 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚22606852 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚51283914 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚28687221 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2422448108032 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚221246278902272 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚220981030320308é ?2Z2ƒ - _wandbruntime‚1973  _runtime‚1973>7à–„ ÿ 2loss/SRqqt2/assignment_loss‚0.7207604130621541 /loss/FRt1/detection_loss‚0.5572350893950833 /loss/FBt1/detection_loss‚0.8156055510557692 %loss/total_loss‚12.58982801262253 _step‚234 /loss/FRt1/assignment_loss‚2.825433784299861 2loss/FBt1/assignment_loss‚0.049468565279703114 1loss/SRqqt2/detection_loss‚0.7710363212881454 epoch‚7 trainer/global_step‚2849 % _timestamp‚1.7844622995056636e+09 .loss/FRt2/detection_loss‚1.342568289465751 0loss/FRt2/assignment_loss‚1.4171619823912984 2loss/SRqqt1/assignment_loss‚0.8675843615868044 1loss/FBt2/assignment_loss‚0.11690593543663784 0loss/SRqqt1/detection_loss‚1.400940729897808 /loss/FBt2/detection_loss‚1.7051269894635135 _runtime‚1973.893753859ì—;! … „ /loss/FBt2/detection_loss‚1.7051269894635135 _runtime‚1973.893753859 2loss/SRqqt2/assignment_loss‚0.7207604130621541 /loss/FRt1/detection_loss‚0.5572350893950833 /loss/FBt1/detection_loss‚0.8156055510557692 %loss/total_loss‚12.58982801262253 _step‚234 /loss/FRt1/assignment_loss‚2.825433784299861 2loss/FBt1/assignment_loss‚0.049468565279703114 1loss/SRqqt2/detection_loss‚0.7710363212881454 epoch‚7 trainer/global_step‚2849 % _timestamp‚1.7844622995056636e+09 .loss/FRt2/detection_loss‚1.342568289465751 0loss/FRt2/assignment_loss‚1.4171619823912984 2loss/SRqqt1/assignment_loss‚0.8675843615868044 1loss/FBt2/assignment_loss‚0.11690593543663784 0loss/SRqqt1/detection_loss‚1.400940729897808êËà5bu† ‚(:ê á÷òÒéýߤ disk.dm-1.in‚0! disk.dm-1.out‚1451.8466796875 disk.rbd0.in‚0' disk.nvme0c0n1.out‚24810.8408203125) disk./.usagePercent‚76.28294884663532% disk./.usageGB‚1171.1339721679688 disk.nvme1c1n1.in‚ 129.765625 disk.nvme1n1.in‚ 129.765625$ memory_percent‚98.81389141082764" proc.memory.rssMB‚ 2964.7578125 cpu‚25.85374553241048 network.sent‚ 4.186828e+06 network.recv‚547862 disk.dm-0.in‚ 1982.2109375" disk.nvme0c0n1.in‚ 1852.4453125) proc.memory.percent‚18.09544563293457 proc.cpu.threads‚17 disk.md0.out‚ 49.93359375# disk.dm-0.out‚38023.60205078125 disk.nvme0n1.in‚ 1852.4453125% disk.nvme0n1.out‚24810.8408203125( disk.nvme1c1n1.out‚14771.95263671875 disk.md0.in‚0 disk.rbd0.out‚0( proc.memory.availableMB‚ 194.33203125& disk.nvme1n1.out‚14771.95263671875ýM*ž‡ ‚(:“ á÷òÒçå½¥ gpu.0.gpu‚55.0 gpu.0.memory‚10, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚48.0 gpu.0.powerWatts‚193.544' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚64.51466666666667 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0]¸€rˆ ‚(:ç ð÷òÒšë̤" proc.memory.rssMB‚ 2964.7578125 disk.rbd0.out‚0( disk.nvme0c0n1.out‚24811.92236328125) proc.memory.percent‚18.09544563293457 cpu‚25.851854437246136 proc.cpu.threads‚17$ disk./.usageGB‚1171.133960723877% disk.nvme1n1.out‚14772.3681640625 disk.md0.out‚ 50.08984375# disk.dm-0.out‚38024.56298828125! disk.dm-1.out‚1452.0009765625 disk.nvme0n1.in‚ 1852.4453125& disk.nvme0n1.out‚24811.92236328125& proc.memory.availableMB‚ 194.328125 network.sent‚ 4.250222e+06 network.recv‚552312 disk.nvme1n1.in‚ 129.765625 disk.md0.in‚0 disk.dm-0.in‚ 1982.2109375 disk.dm-1.in‚0 disk.rbd0.in‚0" disk.nvme0c0n1.in‚ 1852.4453125( disk./.usagePercent‚76.2829481012133 disk.nvme1c1n1.in‚ 129.765625' disk.nvme1c1n1.out‚14772.3681640625$ memory_percent‚98.81391525268555ÆîÝž‰ ‚(:“ ð÷òÒð»þ¤ gpu.0.gpu‚81.0 gpu.0.memory‚13, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚49.0 gpu.0.powerWatts‚220.637' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚73.54566666666666 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0κú2Š - _wandbruntime‚2005  _runtime‚2005 „tÉ‹ à ' lr-AdamW/pg2‚0.00013373175041109356 trainer/global_step‚2899 % _timestamp‚1.7844623312647545e+09 _runtime‚2005.654810977 _step‚235 ' lr-AdamW/pg1‚0.00013373175041109356S&“ÎŒ È ' lr-AdamW/pg2‚0.00013373175041109356 trainer/global_step‚2899 % _timestamp‚1.7844623312647545e+09 _runtime‚2005.654810977 _step‚235 ' lr-AdamW/pg1‚0.00013373175041109356ë5{ëà2 - _wandbruntime‚2005  _runtime‚2005²!y(Ž ÚO VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚227319682065920 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2390963815424 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2391052475392 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2462259698688 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚22980673 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 _runtime‚2005.654877188 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚227319576366592 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚52140767 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2384490550196 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 % _timestamp‚1.7844623312653909e+09 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚15421486 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2462382851072 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚29160096 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚15421482 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚224659721100612 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚227319576366592 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚22980673 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚29170239 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚222275213511056 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚29160096 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚199 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚224928629590528 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚224928612551168 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2391052475392 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚170930702686720 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚6573096 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚22980671 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚195 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 LDDeviceStatsMonitor.on_train_batch_start/segmeç‰â¸nt.small_pool.allocated‚130 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚224928629590528 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚170930485424640 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚168468319835648 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2384403763770 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚227319576366592 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚52140767 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2384490550196 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 % _timestamp‚1.7844623312653909e+09 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚15421486 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2462382851072 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚29160096 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚15421482 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚224659721100612 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚227319576366592 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚22980673 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚29170239 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚222275213511056 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚29160096 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚199 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚224928629590528 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚224928612551168 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2391052475392 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚170930702686720 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚6573096 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚22980671 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚195 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚224928629590528 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚170930485424640 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚168468319835648 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2384403763770 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 IDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚52168375 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚52158216 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚225003779256832 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚225003762217472 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚227395511748608 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚29180073 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚29169916 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚22988302 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2463072184832 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚225003779256832 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚191 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚29169916 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚52168375 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚15426538 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 _runtime‚2006.055112358 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚6575438 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚170987435638784 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚29180073 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6575251 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2385274044908 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 % _timestamp‚1.7844623316665986e+09 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚22988300 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚222349504036448 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚187 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2391838198272 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚222349521075808 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2391749531136 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚224734795120716 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚22988300 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚22001980 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚168524457563648 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚225003762217472 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 221449216 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚227395617455104 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚22001789 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚15426542 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚170987657088000 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚224734691294818 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2391838198272 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚52158216 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048wwû$ ’ éM NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚29180073 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6575251 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2385274044908 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 % _timestamp‚1.7844623316665986e+09 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚22988300 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚222349504036448 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚187 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2391838198272 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚222349521075808 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2391749531136 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚224734795120716 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚22988300 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚22001980 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚168524457563648 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚225003762217472 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 221449216 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚227395617455104 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚22001789 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚15426542 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚170987657088000 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚224734691294818 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2391838198272 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚52158216 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚52168375 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚52158216 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚225003779256832 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚225003762217472 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚227395511748608 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚29180073 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚29169916 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚22988302 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2463072184832 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚225003779256832 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚191 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.la£ßhËrge_pool.peak‚ 11862167552 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚29169916 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚52168375 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚15426538 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 _runtime‚2006.055112358 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚6575438 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚170987435638784 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0íÎ8!z2“ - _wandbruntime‚2006  _runtime‚2006ós“W”  2loss/SRqqt1/assignment_loss‚0.8240915509699823 /loss/FRt1/detection_loss‚0.5352529891413172 /loss/FRt2/detection_loss‚1.4223448416007856 0loss/SRqqt1/detection_loss‚1.288019686428592 % _timestamp‚1.7844623316675298e+09 _runtime‚2006.055663989 /loss/FBt2/detection_loss‚1.4771298542318325 epoch‚7 1loss/FBt1/assignment_loss‚0.10056388500848959 1loss/FBt2/assignment_loss‚0.12539008357922402 /loss/FBt1/detection_loss‚0.8311648389605226 &loss/total_loss‚12.109898499045048 trainer/global_step‚2899 0loss/FRt2/assignment_loss‚1.2903214831033192 2loss/SRqqt2/assignment_loss‚0.6919139244244135 1loss/SRqqt2/detection_loss‚0.7318679022294393 _step‚238 0loss/FRt1/assignment_loss‚2.7918374593671302Ù¼U • † 1loss/FBt1/assignment_loss‚0.10056388500848959 1loss/FBt2/assignment_loss‚0.12539008357922402 /loss/FBt1/detection_loss‚0.8311648389605226 &loss/total_loss‚12.109898499045048 trainer/global_step‚2899 0loss/FRt2/assignment_loss‚1.2903214831033192 2loss/SRqqt2/assignment_loss‚0.6919139244244135 1loss/SRqqt2/detection_loss‚0.7318679022294393 _step‚238 0loss/FRt1/assignment_loss‚2.7918374593671302 2loss/SRqqt1/assignment_loss‚0.8240915509699823 /loss/FRt1/detection_loss‚0.5352529891413172 /loss/FRt2/detection_loss‚1.4223448416007856 0loss/SRqqt1/detection_loss‚1.288019686428592 % _timestamp‚1.7844623316675298e+09 _runtime‚2006.055663989 /loss/FBt2/detection_loss‚1.4771298542318325 epoch‚7 Çq– ‚(:æ ÿ÷òÒã¼ì¤& disk.nvme0n1.out‚24813.35888671875$ disk.nvme1n1.out‚14773.005859375! disk.dm-1.out‚1452.0439453125$ memory_percent‚98.81489276885986 cpu‚25.84985725613775 network.sent‚ 4.267924e+06 network.recv‚555638 disk.nvme1c1n1.in‚ 129.765625 disk.dm-0.in‚ 1982.2109375# disk.dm-0.out‚38025.61572265625" proc.memory.rssMB‚ 2964.7578125 proc.cpu.threads‚17% disk./.usageGB‚1171.1340408325195" disk.nvme0c0n1.in‚ 1852.4453125 disk.nvme1n1.in‚ 129.765625 disk.md0.out‚ 50.546875 disk.dm-1.in‚0) proc.memory.percent‚18.09544563293457) disk./.usagePercent‚76.28295331916753 disk.rbd0.in‚0 disk.rbd0.out‚0 disk.nvme0n1.in‚ 1852.4453125& disk.nvme1c1n1.out‚14773.005859375 disk.md0.in‚0( proc.memory.availableMB‚ 194.16796875( disk.nvme0c0n1.out‚24813.35888671875JdÙbž— ‚(:“ ÿ÷òÒíþŸ¥ gpu.0.gpu‚47.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚47.0 gpu.0.powerWatts‚113.927' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚37.97566666666667 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚03ow˜ ‚(:ì ŽøòÒÏ°Ó¥ disk.nvme1c1n1.in‚ 129.765625( disk.nvme1c1n1.out‚14774.07666015625 disk.nvme1n1.in‚ 129.765625& disk.nvme1n1.out‚14774.07666015625 disk.rbd0.out‚0" disk.nvme0c0n1.in‚ 1852.4453125$ memory_percent‚98.81491661071777) disk./.usagePercent‚76.28295977949183 disk.nvme0n1.in‚ 1852.4453125 disk.md0.out‚ 51.00390625 disk.dm-1.in‚0' proc.memory.availableMB‚ 194.1640625) proc.memory.percent‚18.09544563293457 proc.cpu.threads‚17 network.sent‚ 4.316386e+06 network.recv‚559857 disk.md0.in‚0 disk.rbd0.in‚0" proc.memory.rssMB‚ 2964.7578125% disk./.usageGB‚1171.1341400146484 disk.dm-0.in‚ 1982.2109375# disk.dm-0.out‚38028.10400390625" disk.dm-1.out‚1452.23193359375( disk.nvme0c0n1.out‚24815.94091796875 cpu‚25.84824261027905& disk.nvme0n1.out‚24815.94091796875Â.øãž™ ‚(:“ ŽøòÒ×›œ¦ gpu.0.gpu‚49.0 gpu.0.memory‚10, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚49.0 gpu.0.powerWatts‚227.353' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚75.78433333333334 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚03 Í2š - _wandbruntime‚2037  _runtime‚2037MMËÉ› à trainer/global_step‚2949 % _timestamp‚1.7844623632623353e+09 _runtime‚2037.650711332 _step‚239 ' lr-AdamW/pg1‚0.00013351212841771593 ' lr-AdamW/pg2‚0.00013351212841771593¡Š§Îœ È _runtime‚2037.650711332 _step‚239 ' lr-AdamW/pg1‚0.00013351212841771593 ' lr-AdamW/pg2‚0.00013351212841771593 trainer/global_step‚2949 % _timestamp‚1.7844623632623353e+09ïÈa§ 2 - _wandbruntime‚2037  _runtime‚2037‡KAà'ž ÚO TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 % _timestamp‚1.7844623632632222e+09 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚171275206235648 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2502855814144 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚29652948 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2423729081004 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚228686112905728 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚6682244 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚15674286 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚15674282 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚23362121 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚22356348 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 _runtime‚2037.651712714 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚22356530 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2423642294578 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚23362121 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 121055232 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚228686095866368 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚231116515321344 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚23362123 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚225989739780656 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚231116515321344 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2430402415616 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2430313755648 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 trainer/global_step‚2949 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚29663091 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2502976869376 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚53015069 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2430402415616 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚225989756820016 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚178 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚173777967940096 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚228413485901020 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚231116409622016 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚173778183105024 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 215164928 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 _step‚240 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚231116515321344 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2430402415616 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2430313755648 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 trainer/global_step‚2949 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚29663091 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2502976869376 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚53015069 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2430402415616 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚225989756820016 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚178 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚173777967940096 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚228413485901020 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚231116409622016 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚173778183105024 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 215164928 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 _step‚240 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 % _timestamp‚1.7844623632632222e+09 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚171275206235648 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2502855814144 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚29652948 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2423729081004 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚228686112905728 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚6682244 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚15674286 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚15674282 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚23362121 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚22356348 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 _runtime‚2037.651712714 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚22356530 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2423642294578 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚23362121 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 121055232 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚228686095866368 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚231116515321344 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚23362123 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509ð<šzì2  - _wandbruntime‚2038  _runtime‚2038/Ôkj¡ ãM N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚22363618 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 _runtime‚2038.058833583 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2431188138496 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6684280 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚53042677 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚15679338 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2431099471360 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚22363774 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 trainer/global_step‚2949 _step‚241 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚228761245532672 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚152 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 112659456 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚15679342 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚53042677 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.currenÊŒ^˜t‚10157 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚29662768 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚228761262572032 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2503783450112 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚29672925 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚226064030306048 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚23369752 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2431099471360 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚53032518 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚23369750 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 206769152 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚29662768 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2503670790656 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2431188138496 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚228488456095226 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚226064047345408 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚171331343963648 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2424425789178 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 $ _timestamp‚1.784462363670208e+09 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚53032518 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚29672925 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760{nâÏî&¢ èM \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2424512575716 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚228761245532672 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚152 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 112659456 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚15679342 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚53042677 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚29662768 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚228761262572032 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2503783450112 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚29672925 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚226064030306048 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚23369752 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2431099471360 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚53032518 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚23369750 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 206769152 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚29662768 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2503670790656 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2431188138496 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚228488456095226 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚226064047345408 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚171331343963648 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2424425789178 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 $ _timestamp‚1.784462363670208e+09 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚53032518 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚29672925 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚22363618 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 _runtime‚2038.058833583 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2431188138496 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6684280 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚53042677 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚15679338 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2431099471360 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚22363774 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 trainer/global_step‚2949 _step‚241 EDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚15927086 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚232443579181568 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚23743571 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚6790075 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2469663695872 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚23743573 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚6789905 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚176625596365312 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚232167250701428 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚229704266050256 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 % _timestamp‚1.7844623965041819e+09 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚234913348576768 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚234913348576768 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 _step‚244 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2469752355840 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚234913242877440 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2543382674432 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚15927082 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚232443596220928 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚174 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚23743573 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚232443579181568 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2469663695872 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚30155943 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 trainer/global_step‚2999 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚176625381200384 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚53889371 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚22716987 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 215164928 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚174081998525952 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚22717161 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚234913242877440 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682Bì2jб ÞO cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2543503729664 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2469752355840 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚174 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚23743573 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚232443579181568 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2469663695872 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚30155943 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 trainer/global_step‚2999 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚176625381200384 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚53889371 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚22716987 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 215164928 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚174081998525952 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 2726297±ÿô_60 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚22717161 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚234913242877440 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚15927086 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚232443579181568 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚23743571 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚6790075 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2469663695872 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚23743573 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚6789905 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚176625596365312 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚232167250701428 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚229704266050256 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 % _timestamp‚1.7844623965041819e+09 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚234913348576768 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚234913348576768 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 _step‚244 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2469752355840 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚234913242877440 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2543382674432 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚15927082 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚232443596220928 IDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚23751200 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6792119 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 131533824 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚232518728847872 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚22724442 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2463751106524 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 _step‚245 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚30155620 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚53916979 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚30165777 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚232518745887232 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚181 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚6792300 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚22724257 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚234989283965952 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚53906820 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚30165777 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚30155620 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚232242220895634 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚229778556575648 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚53906820 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚234989283965952 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚174138230363648 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2544325121536 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2544193587712 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760Vüï&´ éM ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚232518745887232 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚181 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚6792300 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚22724257 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚234989283965952 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚53906820 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚30165777 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚30155620 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚232242220895634 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚229778556575648 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚53906820 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚234989283965952 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚174138230363648 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2544325121536 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2544193587712 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2463664319986 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 trainer/global_step‚2999 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2470449411584 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 _runtime‚2071.314992228 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚232518745887232 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚23751200 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6792119 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 131533824 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚232518728847872 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚22724442 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2463751106524 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 _step‚245 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚30155620 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚53916979 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚30165777 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279õ2K›\2µ - _wandbruntime‚2071  _runtime‚2071¼ÈDܶ € trainer/global_step‚2999 _runtime‚ 2071.31628638 .loss/FRt1/detection_loss‚0.566019892761032 /loss/FRt2/detection_loss‚1.4516019865607435 % _timestamp‚1.7844623969280343e+09 _step‚246 0loss/FRt1/assignment_loss‚2.7685658077935913 2loss/SRqqt1/assignment_loss‚0.8550173627674147 2loss/SRqqt2/assignment_loss‚0.7247517202015081 1loss/FBt2/assignment_loss‚0.16461884336508534 /loss/FBt1/detection_loss‚0.6275731139949959 epoch‚7 0loss/FRt2/assignment_loss‚1.2042694205698143 1loss/FBt1/assignment_loss‚0.08145991136959196 1loss/SRqqt1/detection_loss‚1.2426377950988743 1loss/SRqqt2/detection_loss‚0.7076192917934847 /loss/FBt2/detection_loss‚1.3661519656521512 &loss/total_loss‚11.760287111928287¦zV · … /loss/FBt2/detection_loss‚1.3661519656521512 &loss/total_loss‚11.760287111928287 trainer/global_step‚2999 _runtime‚ 2071.31628638 .loss/FRt1/detection_loss‚0.566019892761032 /loss/FRt2/detection_loss‚1.4516019865607435 % _timestamp‚1.7844623969280343e+09 _step‚246 0loss/FRt1/assignment_loss‚2.7685658077935913 2loss/SRqqt1/assignment_loss‚0.8550173627674147 2loss/SRqqt2/assignment_loss‚0.7247517202015081 1loss/FBt2/assignment_loss‚0.16461884336508534 /loss/FBt1/detection_loss‚0.6275731139949959 epoch‚7 0loss/FRt2/assignment_loss‚1.2042694205698143 1loss/FBt1/assignment_loss‚0.08145991136959196 1loss/SRqqt1/detection_loss‚1.2426377950988743 1loss/SRqqt2/detection_loss‚0.7076192917934847öW›Ë'y¸ ‚(:î ÊøòÒâÚ¥ disk.md0.out‚ 52.41796875" disk.dm-1.out‚1453.00048828125 disk.rbd0.in‚0 disk.rbd0.out‚0 disk.nvme1c1n1.in‚ 129.765625) proc.memory.percent‚18.09544563293457 disk.nvme1n1.in‚ 129.765625" proc.memory.rssMB‚ 2964.7578125 network.sent‚ 4.406154e+06 disk.md0.in‚0 disk.dm-0.in‚ 1982.2109375 disk.nvme0n1.in‚ 1852.4453125( disk.nvme1c1n1.out‚14795.91552734375$ memory_percent‚98.80983829498291 cpu‚25.844851989775172 proc.cpu.threads‚17) disk./.usagePercent‚76.34452443386091% disk./.usageGB‚1172.0793113708496# disk.dm-0.out‚38696.56494140625 disk.dm-1.in‚0" disk.nvme0c0n1.in‚ 1852.4453125& disk.nvme0n1.out‚25466.37841796875& disk.nvme1n1.out‚14795.91552734375( proc.memory.availableMB‚ 194.99609375( disk.nvme0c0n1.out‚25466.37841796875 network.recv‚573780£0o5¹ ‚(:’ ÊøòÒ®ÉÄ¥ gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚47.0 gpu.0.powerWatts‚113.523' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚37.840999999999994 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0HH˜º ‚(:’ ÙøòÒ´êФ gpu.0.gpu‚38.0 gpu.0.memory‚6, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚48.0 gpu.0.powerWatts‚220.877' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚73.62566666666666 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚06ÐÌq» ‚(:æ ÙøòÒêüç¤" disk.nvme0c0n1.in‚ 1852.4453125' disk.nvme0c0n1.out‚25773.9619140625% disk.nvme0n1.out‚25773.9619140625" proc.memory.rssMB‚ 2964.7578125 cpu‚25.843003455283007$ disk./.usageGB‚1172.082332611084 disk.nvme1n1.in‚ 129.765625% disk.nvme1n1.out‚14796.7763671875# disk.dm-0.out‚39003.70556640625 disk.rbd0.in‚0$ memory_percent‚98.81136417388916 proc.cpu.threads‚17 network.recv‚577657) disk./.usagePercent‚76.34472122527822 disk.nvme1c1n1.in‚ 129.765625' disk.nvme1c1n1.out‚14796.7763671875 disk.rbd0.out‚0( proc.memory.availableMB‚ 194.74609375) proc.memory.percent‚18.09544563293457 network.sent‚ 4.45756e+06 disk.md0.out‚ 52.921875 disk.dm-0.in‚ 1982.2109375 disk.nvme0n1.in‚ 1852.4453125 disk.md0.in‚0 disk.dm-1.in‚0" disk.dm-1.out‚1453.22314453125÷¨f2¼ - _wandbruntime‚2103  _runtime‚2103£.¶È½  ' lr-AdamW/pg1‚0.00013306058803397387 ' lr-AdamW/pg2‚0.00013306058803397387 trainer/global_step‚3049 % _timestamp‚1.7844624290152998e+09 _runtime‚ 2103.40372019 _step‚247e@­-; Ç % _timestamp‚1.7844624290152998e+09 _runtime‚ 2103.40372019 _step‚247 ' lr-AdamW/pg1‚0.00013306058803397387 ' lr-AdamW/pg2‚0.00013306058803397387 trainer/global_step‚3049÷ƒw1,2¿ - _wandbruntime‚2103  _runtime‚2103¸_£À ÚO WIDeviceS7˜5Ï'tatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚235921015501836 _step‚248 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚236201062496768 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2509102296064 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚16179886 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2583842884608 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚30638652 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2509013636096 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2502206142620 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚24125021 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚30638652 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚172 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚54773818 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚235920911676050 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 225650688 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚238710076132864 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚30648795 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚238710181832192 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚236201079536128 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2502119356194 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2509013636096 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚23077307 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚238710076132864 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2509102296064 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚30648795 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚179472953461248 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚24125021 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚176 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚179472727810560 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2583974425600 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚233418792319856 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚16179882 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚236201079536128 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚238710181832192 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚23077131 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _runtime‚2103.407818917 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚24125023 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 131540992 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚6897249 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2509102296064 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚16179886 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2583842884608 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚30638652 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2509013636096 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2502206142620 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚24125021 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚30638652 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚172 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚54773818 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚235920911676050 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 225650688 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚238710076132864 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚30648795 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚238710181832192 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚236201079536128 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2502119356194 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2509013636096 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚23077307 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚238710076132864 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2509102296064 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚30648795 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚179472953461248 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚24125021 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚176 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚179472727810560 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2583974425600 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚233418792319856 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚16179882 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚236201079536128 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚238710181832192 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚23077131 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _runtime‚2103.407818917 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚24125023 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 131540992 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚6897249 IDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚23084204 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚54791281 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 _step‚249 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚236276212163072 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚24132652 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚30648472 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚236276229202432 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚30648472 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚23084371 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚24132650 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚30658629 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚16184938 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚236276212163072 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚24132650 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 _runtime‚2103.812145058 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚163 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚238786117221376 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 % _timestamp‚1.7844624294236178e+09 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2509799351808 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚54791281 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2584650783232 trainer/global_step‚3049 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚233493082845248 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚24132652 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2502989637332 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚54781122 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6899266 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 221449216 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚238786011514880 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚179529894886400 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚179529673437184 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚235996089521940 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚233493099884608 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚238786117221376 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚6899429 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812.á| Ä éM T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚238786117221376 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚6899429 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚30658629 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚167 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚176945116763648 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚16184942 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚23084204 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚54791281 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 _step‚249 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚236276212163072 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚24132652 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚30648472 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚236276229202432 PBDeviceSta9ƒ ¯âtsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚30648472 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚23084371 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚24132650 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚30658629 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚16184938 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚236276212163072 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚24132650 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 _runtime‚2103.812145058 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚163 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚238786117221376 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 % _timestamp‚1.7844624294236178e+09 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2509799351808 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚54791281 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2584650783232 trainer/global_step‚3049 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚233493082845248 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚24132652 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2502989637332 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚54781122 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚6899266 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 221449216 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚238786011514880 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚179529894886400 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚179529673437184 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚235996089521940 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚233493099884608 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552ù [Ê52Å - _wandbruntime‚2103  _runtime‚2103Ùu-©Æ  0loss/FRt2/assignment_loss‚1.2151981380133503 2loss/SRqqt1/assignment_loss‚0.6545847094280124 /loss/FRt1/detection_loss‚0.5779824750992999 1loss/SRqqt2/detection_loss‚0.7727058401482331 epoch‚7 _step‚250 /loss/FBt2/detection_loss‚1.4807738831071402 % _timestamp‚1.7844624294246056e+09 2loss/SRqqt2/assignment_loss‚0.6965794391368103 1loss/FBt1/assignment_loss‚0.07882188593169911 .loss/FRt2/detection_loss‚1.391064561238377 1loss/SRqqt1/detection_loss‚1.2892938315519684 &loss/total_loss‚11.969043641283847 1loss/FBt2/assignment_loss‚0.11135378465153735 /loss/FBt1/detection_loss‚0.8575780124368215 trainer/global_step‚3049 _runtime‚2103.812794589 0loss/FRt1/assignment_loss‚2.8431070805405976ÏiÛÅ Ç † 1loss/SRqqt1/detection_loss‚1.2892938315519684 &loss/total_loss‚11.969043641283847 1loss/FBt2/assignment_loss‚0.11135378465153735 /loss/FBt1/detection_loss‚0.8575780124368215 trainer/global_step‚3049 _runtime‚2103.812794589 0loss/FRt1/assignment_loss‚2.8431070805405976 0loss/FRt2/assignment_loss‚1.2151981380133503 2loss/SRqqt1/assignment_loss‚0.6545847094280124 /loss/FRt1/detection_loss‚0.5779824750992999 1loss/SRqqt2/detection_loss‚0.7727058401482331 epoch‚7 _step‚250 /loss/FBt2/detection_loss‚1.4807738831071402 % _timestamp‚1.7844624294246056e+09 2loss/SRqqt2/assignment_loss‚0.6965794391368103 1loss/FBt1/assignment_loss‚0.07882188593169911 .loss/FRt2/detection_loss‚1.391064561238377úâ¾pÈ ‚(:å èøòÒ½Æî¤ network.sent‚ 4.460482e+06% disk./.usageGB‚1172.0823516845703" disk.nvme0c0n1.in‚ 1852.4453125& disk.nvme0c0n1.out‚25778.865234375$ disk.nvme0n1.out‚25778.865234375 disk.nvme1c1n1.in‚ 129.765625 disk.nvme1n1.in‚ 129.765625 disk.nvme0n1.in‚ 1852.4453125 disk.md0.in‚0! disk.dm-0.out‚39008.681640625' proc.memory.availableMB‚ 194.9296875" proc.memory.rssMB‚ 2964.7578125) proc.memory.percent‚18.09544563293457) disk./.usagePercent‚76.34472246764828 disk.rbd0.in‚0& disk.nvme1n1.out‚14797.66943359375 disk.dm-0.in‚ 1982.2109375 cpu‚25.841172127035986 network.recv‚581344 disk.dm-1.in‚0" disk.dm-1.out‚1453.41552734375 disk.rbd0.out‚0 disk.md0.out‚ 53.203125$ memory_percent‚98.81024360656738 proc.cpu.threads‚17( disk.nvme1c1n1.out‚14797.66943359375C²Ê×’É ‚(:‡ èøòÒ֢Ϧ gpu.0.gpu‚49.0 gpu.0.memory‚8, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚49.0 gpu.0.powerWatts‚224.253' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚74.751 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0N*æCpÊ ‚(:å ÷øòÒ³²î¤& disk.nvme1n1.out‚14798.04150390625" disk.nvme0c0n1.in‚ 1852.4453125' disk.nvme0c0n1.out‚25782.6787109375" proc.memory.rssMB‚ 2964.7578125) proc.memory.percent‚18.09544563293457 network.recv‚585604 disk.nvme0n1.in‚ 1852.4453125( disk.nvme1c1n1.out‚14798.04150390625 disk.nvme1n1.in‚ 129.765625" disk.dm-1.out‚1453.43505859375 disk.rbd0.out‚0# memory_percent‚98.8102912902832 cpu‚25.839834329709532 network.sent‚ 4.525015e+06% disk.nvme0n1.out‚25782.6787109375 disk.md0.in‚0 disk.md0.out‚ 53.453125 disk.dm-1.in‚0 proc.cpu.threads‚17) disk./.usagePercent‚76.34472172222625 disk.dm-0.in‚ 1982.2109375! disk.dm-0.out‚39012.298828125 disk.rbd0.in‚0& proc.memory.availableMB‚ 194.921875% disk./.usageGB‚1172.0823402404785 disk.nvme1c1n1.in‚ 129.765625’t ©ŸË ‚(:” ÷øòÒÎüž¥ gpu.0.gpu‚91.0 gpu.0.memory‚22, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚48.0 gpu.0.powerWatts‚114.255' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚38.084999999999994 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Æð&2Ì - _wandbruntime‚2135  _runtime‚2135»è,SÉÍ Ã trainer/global_step‚3099 % _timestamp‚1.7844624613726835e+09 _runtime‚2135.762476732 _step‚251 ' lr-AdamW/pg1‚0.00013282869864520444 ' lr-AdamW/pg2‚0.00013282869864520444¿*ešÎÎ È ' lr-AdamW/pg2‚0.00013282869864520444 trainer/global_step‚3099 % _timestamp‚1.7844624613726835e+09 _runtime‚2135.762476732 _step‚251 ' lr-AdamW/pg1‚0.00013282869864520444û²‡Ÿl2Ï - _wandbruntime‚2135  _runtime‚2135Hz­†à'Ð ÚO L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚31131504 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚24506473 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚242507015087616 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚179695771325952 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚55648120 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚16432682 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚24506473 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚31141647 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚242507015087616 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚31131504 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚55637975 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚24506471 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2548363576320 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 _runtime‚2135.762540322 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2548452236288 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚239958562851328 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚7004083 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚239674780302244 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚237133318589456 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 227747840 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 133638144 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚24506471 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2548452236288 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2624434536448 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚239958545811968 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚31141647 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2548363576320 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚179695865435648 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2624300898304 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 trainer/global_step‚3099 % _timestamp‚1.7844624613734024e+09 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚242506909388288 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚23436769 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚239674676476458 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚242507015087616 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚31131504 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚55637975 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚24506471 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2548363576320 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 _runtime‚2135.762540322 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2548452236288 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚239958562851328 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚7004083 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚239674780302244 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚237133318589456 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 227747840 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 133638144 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚24506471 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2548452236288 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2624434536448 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚239958545811968 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚31141647 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2548363576320 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚179695865435648 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2624300898304 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 trainer/global_step‚3099 % _timestamp‚1.7844624613734024e+09 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚242506909388288 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚170393«²`Z60 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚23436769ü=³Ì¼2Ò - _wandbruntime‚2136  _runtime‚2136íK®‚ê&Ó äM NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚31151481 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚182377237465088 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚239749854322348 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2542141381602 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚16437742 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚31151481 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚24514100 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚23443983 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 123145216 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 _step‚253 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚240033695478272 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚55665583 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚169 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2549149292032 trainer/global_step‚3099 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚55655424 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2625111156224 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚239749750496450 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚31141324 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚179752003163648 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚240033712517632 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚237207609114848 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚242582950476800 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2542228168140 _runtime‚2136.165581271 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚23443814 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚182377020210176 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚24514102 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚16437738 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚240033712517632 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2549237959168 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 % _timestamp‚1.7844624617770605e+09 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚55665583 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚55655424 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚240033695478272 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚242582844770304 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚165 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚237207626154208·ð ï&Ô éM RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 % _timestamp‚1.7844624617770605e+09 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚55665583 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚55655424 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚240033695478272 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚242582844770304 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚165 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚237207626154208 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚31151481 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚182377237465088 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚239749854322348 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2542141381602 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚16437742 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚31151481 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚24514100 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚23443983 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 123145216 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 _step‚253 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚240033695478272 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚55665583 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚169 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2549149292032 trainer/global_step‚3099 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚55655424 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2625111156224 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚239749750496450 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚31141324 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚179752003163648 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚240033712517632 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚237207609114848 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚242582950476800 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2542228168140 _runtime‚2136.165581271 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚23443814 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚182377020210176 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚24514102 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚16437738 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚240033712517632 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2549237959168 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760ýÆr¦¦2Õ - _wandbruntime‚2136  _runtime‚2136ó³/®Ö € /loss/FRt1/assignment_loss‚2.736325595630418 2loss/SRqqt2/assignment_loss‚0.7386409056875683 1loss/FBt1/assignment_loss‚0.09345236584608443 1loss/SRqqt1/detection_loss‚1.2877817090553374 1loss/SRqqt2/detection_loss‚0.7091395905363104 _runtime‚2136.166151322 0loss/FRt2/assignment_loss‚1.3041619770739177 2loss/SRqqt1/assignment_loss‚0.7499045291185473 1loss/FBt2/assignment_loss‚0.14960773088693569 trainer/global_step‚3099 _step‚254 epoch‚7 /loss/FRt2/detection_loss‚1.3765815028945487 /loss/FBt1/detection_loss‚1.0183755087733752 &loss/total_loss‚12.399109186848726 /loss/FRt1/detection_loss‚0.5852313031111174 .loss/FBt2/detection_loss‚1.649906468234564 % _timestamp‚1.7844624617780929e+09ÕÒðÙ × … /loss/FRt1/detection_loss‚0.5852313031111174 .loss/FBt2/detection_loss‚1.649906468234564 % _timestamp‚1.7844624617780929e+09 /loss/FRt1/assignment_loss‚2.736325595630418 2loss/SRqqt2/assignment_loss‚0.7386409056875683 1loss/FBt1/assignment_loss‚0.09345236584608443 1loss/SRqqt1/detection_loss‚1.2877817090553374 1loss/SRqqt2/detection_loss‚0.7091395905363104 _runtime‚2136.166151322 0loss/FRt2/assignment_loss‚1.3041619770739177 2loss/SRqqt1/assignment_loss‚0.7499045291185473 1loss/FBt2/assignment_loss‚0.14960773088693569 trainer/global_step‚3099 _step‚254 epoch‚7 /loss/FRt2/detection_loss‚1.3765815028945487 /loss/FBt1/detection_loss‚1.0183755087733752 &loss/total_loss‚12.399109186848726þôíç£fØ ‚(:Û †ùòÒ“•ë¤ disk.nvme1n1.in‚ 129.765625 disk.md0.out‚ 54.00390625# disk.dm-0.out‚39014.46337890625 disk.rbd0.out‚0% disk.nvme1c1n1.out‚14799.20703125" disk.dm-1.out‚1453.53759765625" proc.memory.rssMB‚ 2964.7578125 proc.cpu.threads‚17 network.sent‚ 4.527937e+06 network.recv‚589096) disk./.usagePercent‚76.34476470823029% disk./.usageGB‚1172.0830001831055 disk.md0.in‚0' proc.memory.availableMB‚ 194.8828125) proc.memory.percent‚18.09544563293457" disk.nvme0c0n1.in‚ 1852.4453125 disk.nvme0n1.in‚ 1852.4453125# disk.nvme1n1.out‚14799.20703125 disk.dm-0.in‚ 1982.2109375 disk.dm-1.in‚0 disk.rbd0.in‚0# memory_percent‚98.8105297088623 cpu‚25.83816826312997# disk.nvme0c0n1.out‚ 25784.953125! disk.nvme0n1.out‚ 25784.953125 disk.nvme1c1n1.in‚ 129.765625ošY’Ù ‚(:‡ †ùòÒÎþË¥ gpu.0.gpu‚48.0 gpu.0.memory‚8, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚48.0 gpu.0.powerWatts‚223.551' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚74.517 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0yâ¦GžÚ ‚(:“ •ùòҘ赤 gpu.0.gpu‚69.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚49.0 gpu.0.powerWatts‚210.807' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚70.26899999999999 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Æ<5wÛ ‚(:ì •ùòÒ̃ø¤$ memory_percent‚98.81055355072021) proc.memory.percent‚18.09544563293457 cpu‚25.83617703414763) disk./.usagePercent‚76.16781866175793% disk./.usageGB‚1169.3664360046387 disk.rbd0.in‚0 disk.nvme1c1n1.in‚ 129.765625 disk.nvme1n1.in‚ 129.765625 network.recv‚593317( disk.nvme0c0n1.out‚25785.98095703125& disk.nvme0n1.out‚25785.98095703125( disk.nvme1c1n1.out‚14799.82080078125 disk.dm-1.in‚0" disk.dm-1.out‚1453.59228515625 proc.cpu.threads‚17 network.sent‚ 4.59143e+06& disk.nvme1n1.out‚14799.82080078125 disk.md0.in‚0 disk.dm-0.in‚ 1982.2109375# disk.dm-0.out‚39015.40869140625( proc.memory.availableMB‚ 194.87890625" proc.memory.rssMB‚ 2964.7578125 disk.rbd0.out‚0" disk.nvme0c0n1.in‚ 1852.4453125 disk.nvme0n1.in‚ 1852.4453125 disk.md0.out‚ 54.30078125{H-j2Ü - _wandbruntime‚2167  _runtime‚2167É•ŽyÉÝ Ã ' lr-AdamW/pg1‚0.00013259274956303128 ' lr-AdamW/pg2‚0.00013259274956303128 trainer/global_step‚3149 % _timestamp‚1.7844624934388225e+09 _runtime‚2167.827262588 _step‚255šïíµÎÞ È ' lr-AdamW/pg1‚0.00013259274956303128 ' lr-AdamW/pg2‚0.00013259274956303128 trainer/global_step‚3149 % _timestamp‚1.7844624934388225e+09 _runtime‚2167.827262588 _step‚255ÿ?´ 2ß - _wandbruntime‚2167  _runtime‚2167”üì$àà ÚO TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚24887923 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚16685482 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚240847861898416 _step‚256 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚179 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚243716029127168 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚243716029127168 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚7110800 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 229844992 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚246303848343040 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚24887921 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚31624356 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚185167486311936 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚182502751835648 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚7110625 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚56522422 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚243428441276866 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚56512277 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2587802176512 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚182502657725952 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2664828585984 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2664964321280 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚24887923 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚243716046166528 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2587802176512 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚24887921 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 trainer/global_step‚3149 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚16685486 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚246303848343040 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚31634499 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚243428545102652 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚240847844859056 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚185167716156928 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 135735296 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚246303742643712 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 _runtime‚2167.827849249 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚23796107 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚175 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚243716046166528 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2580683204236 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 QDDevi?ˆSë ceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚23796286 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 % _timestamp‚1.7844624934395869e+09 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚31624356 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2587713516544 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚246303742643712 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚185167716156928 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 135735296 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚246303742643712 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 _runtime‚2167.827849249 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚23796107 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚175 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚243716046166528 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2580683204236 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚23796286 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 % _timestamp‚1.7844624934395869e+09 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚31624356 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2587713516544 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚246303742643712 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 IDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚7112635 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚56539885 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2588587899392 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚23803346 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚243791195832832 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚24895552 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚31644333 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚243503515296858 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚240922152423808 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚246379783732224 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚185224661383168 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2588499232256 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚240922135384448 _runtime‚ 2168.53977894 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚185224431545344 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚243791178793472 % _timestamp‚1.7844624941510575e+09 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚23803173 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚169 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚243503619122756 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2581379912410 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2581466698948 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚24895552 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2588499232256 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚56529726 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚182558889563648 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2665636091392 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2588587899392 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚173 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚246379678025728 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159mnî&ä èM FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚23803173 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚169 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚243503619122756 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2581379912410 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2581466698948 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚24895552 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2588499232256 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚56529726 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚182558889563648 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2665636091392 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2588587899392 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚173 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚246379678025728 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 trainer/global_step‚3149 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚7112635 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚56539885 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2588587899392 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚23803346 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚243791195832832 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚24895552 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚31644333 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚243503515296858 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚240922152423808 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚246379783732224 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚185224661383168 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2588499232256 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 147779584 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚240922135384448 _runtime‚ 2168.53977894 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚185224431545344 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚243791178793472 % _timestamp‚1.7844624941510575e+09 E‹²Åí ¿ % lr-AdamW/pg1‚0.000132352755942023 % lr-AdamW/pg2‚0.000132352755942023 trainer/global_step‚3199 % _timestamp‚1.7844625256211336e+09 _runtime‚2200.011021799 _step‚259ƒ‰Êî Ä _step‚259 % lr-AdamW/pg1‚0.000132352755942023 % lr-AdamW/pg2‚0.000132352755942023 trainer/global_step‚3199 % _timestamp‚1.7844625256211336e+09 _runtime‚2200.011021799ƒ ß|¶2ï - _wandbruntime‚2200  _runtime‚2200¶Od…à'ð ÚO _runtime‚2200.011583638 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚247473529481728 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 223553536 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚247182309903060 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚24155173 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚25269371 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚250100575899136 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚247182206077274 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚174 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚16938282 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚7217061 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 129443840 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚170 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2619834948618 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚244562388168016 % _timestamp‚1.7844625256232922e+09 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚25269373 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚247473512442368 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚57396724 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚32127351 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 trainer/global_step‚3199 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚25269371 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚188015064585216 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2705426349568 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2705296905728 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚57386579 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚32117208 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚250100681598464 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚185309544125952 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚32127351 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚250100575899136 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚250100681598464 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚247473512442368 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2627152116736 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚247473529481728 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚7216891 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2619921735044 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚24155347 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚25269373 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 _step‚260 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2627063456768 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚244562371128656 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚16938286 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚185309638235648 Iµå'ñ ßO E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 129443840 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚170 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2619834948618 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚244562388168016 % _timestamp‚1.7844625256232922e+09 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚25269373 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚247473512442368 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚57396724 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚32127351 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 147779584 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 trainer/global_step‚3199 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚25269371 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚188015064585216 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2705426349568 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2705296905728 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚57386579 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚32117208 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚250100681598464 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚185309544125952 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚32127351 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚250100575899136 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚250100681598464 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚247473512442368 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2627152116736 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚247473529481728 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚7216891 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2619921735044 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚24155347 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚25269373 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 _step‚260 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2627063456768 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚244562371128656 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚16938286 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚185309638235648 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚25269371 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚250100575899136 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚247182206077274 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚174 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚16938282 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚7217061„_÷P2ò - _wandbruntime‚2200  _runtime‚22006ÎêXIó äM H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 _runtime‚2200.415338955 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚32127028 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚185365681853952 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 trainer/global_step‚3199 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2627937839616 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚247548679148032 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2627849172480 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚57414187 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚250176616987648 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚179 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚16943342 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚24162582 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚250176616987648 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚16943338 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚57404028 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚32137185 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2627937839616 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚244636661654048 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚247548679148032 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚24162403 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 137825280 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 231934976 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2706239328256 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2627849172480 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2620618443218 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚57404028 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚175 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚25277002 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚25277002 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 trainer/global_step‚3199 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2627937839616 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚247548679148032 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2627849172480 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚57414187 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚250176616987648 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚179 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚16943342 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚24162582 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚250176616987648 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚16943338 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚57404028 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚32137185 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2627937839616 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚244636661654048 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚247548679148032 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚24162403 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 137825280 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 231934976 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2706239328256 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2627849172480 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2620618443218…¤6åJ2õ - _wandbruntime‚2200  _runtime‚2200ãªm¦ö € 0loss/FRt2/assignment_loss‚1.3319822216243076 2loss/SRqqt2/assignment_loss‚0.6759082253686384 /loss/FRt2/detection_loss‚1.3216376427234264 1loss/SRqqt1/detection_loss‚1.2079046233636306 _runtime‚2200.415842585 0loss/FRt1/assignment_loss‚2.7238296554449204 2loss/SRqqt1/assignment_loss‚0.7152234285165098 1loss/FBt1/assignment_loss‚0.10360466089700367 epoch‚7 _step‚262 1loss/FBt2/assignment_loss‚0.07177213184013453 0loss/SRqqt2/detection_loss‚0.706097956791151 /loss/FBt1/detection_loss‚0.6294263997397018 trainer/global_step‚3199 $ _timestamp‚1.784462526027744e+09 /loss/FRt1/detection_loss‚0.6125645142675518 /loss/FBt2/detection_loss‚1.2147102606083666 &loss/total_loss‚11.314661721185342«u¢ ÷ … 1loss/FBt2/assignment_loss‚0.07177213184013453 0loss/SRqqt2/detection_loss‚0.706097956791151 /loss/FBt1/detection_loss‚0.6294263997397018 trainer/global_step‚3199 $ _timestamp‚1.784462526027744e+09 /loss/FRt1/detection_loss‚0.6125645142675518 /loss/FBt2/detection_loss‚1.2147102606083666 &loss/total_loss‚11.314661721185342 0loss/FRt2/assignment_loss‚1.3319822216243076 2loss/SRqqt2/assignment_loss‚0.6759082253686384 /loss/FRt2/detection_loss‚1.3216376427234264 1loss/SRqqt1/detection_loss‚1.2079046233636306 _runtime‚2200.415842585 0loss/FRt1/assignment_loss‚2.7238296554449204 2loss/SRqqt1/assignment_loss‚0.7152234285165098 1loss/FBt1/assignment_loss‚0.10360466089700367 epoch‚7 _step‚262†ÊXä7oø ‚(:ä ÂùòÒ” Õ¤ disk.rbd0.out‚0% disk.nvme0n1.out‚25801.3173828125 disk.md0.in‚0 disk.rbd0.in‚0" disk.nvme0c0n1.in‚ 1852.4453125 disk.nvme0n1.in‚ 1852.4453125" memory_percent‚98.710036277771( proc.memory.availableMB‚ 211.34765625 proc.memory.rssMB‚ 2947.71875 network.recv‚603607% disk./.usageGB‚1169.1273193359375 disk.dm-0.in‚ 1982.2109375) proc.memory.percent‚17.99144744873047 cpu‚25.839281307273826 proc.cpu.threads‚17 network.sent‚ 4.661488e+06) disk./.usagePercent‚76.15224356527845 disk.nvme1n1.in‚ 129.765625% disk.nvme1n1.out‚14803.2412109375" disk.dm-0.out‚39031.5810546875 disk.dm-1.in‚0' disk.nvme0c0n1.out‚25801.3173828125 disk.nvme1c1n1.in‚ 129.765625' disk.nvme1c1n1.out‚14803.2412109375 disk.md0.out‚ 55.34765625! disk.dm-1.out‚1453.9423828125¾Ç0Šœù ‚(:‘ ÂùòÒ´ûéÄ gpu.0.gpu‚1.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚47.0 gpu.0.powerWatts‚113.396' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚37.79866666666667 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0î4 A2ú - _wandbruntime‚2213  _runtime‚2213ú1€Hö=û ð{ 9"Purity/*FRt1SRqqt0FBt/SRqqt_purity‚0.6755766868591309 9"Purity/0FRt*SRqqt1FBt/event_purity‚0.8309464454650879 :"Purity/0FRt2SRqqt*FBt/event_purity‚0.37108752131462097 7 Purity/1FRt*SRqqt2FBt/FBt_purity‚0.9815192222595215 7 Purity/1FRt2SRqqt*FBt/FRt_purity‚0.0668833926320076 8 Purity/1FRt2SRqqt1FBt/FRt_purity‚0.10000000149011612 ;"Purity/2FRt*SRqqt*FBt/event_purity‚0.022109752520918846 particle/accuracy_1_of_4‚1 9"Purity/0FRt*SRqqt2FBt/event_purity‚0.8800085783004761 =&Purity/0FRt0SRqqt1FBt/event_proportion‚0.0313895083963871 7 Purity/0FRt0SRqqt2FBt/FBt_purity‚0.9672878980636597 7 Purity/1FRt1SRqqt*FBt/FBt_purity‚0.9872215986251831 ?&Purity/2FRt1SRqqt0FBt/event_proportion‚0.011928013525903225 @&Purity/2FRt1SRqqt2FBt/event_proportion‚2.325148852833081e-05 ("Purity/2FRt1SRqqt2FBt/event_purity‚0 & Purity/2FRt1SRqqt2FBt/FBt_purity‚1 8 Purity/*FRt*SRqqt*FBt/FRt_purity‚0.18540947139263153 7 Purity/*FRt*SRqqt1FBt/FRt_purity‚0.5140624642372131 7 Purity/*FRt*SRqqt2FBt/FRt_purity‚0.8554421663284302 7 Purity/0FRt1SRqqt1FBt/FBt_purity‚0.9862236976623535 & Purity/2FRt*SRqqt2FBt/FBt_purity‚1 ,&Purity/*FRt2SRqqt2FBt/event_proportion‚0 9"Purity/1FRt*SRqqt1FBt/event_purity‚0.3836148679256439 ("Purity/2FRt2SRqqt*FBt/event_purity‚0 8 Purity/2FRt2SRqqt*FBt/FRt_purity‚0.05001583695411682 =&Purity/0FRt*SRqqt1FBt/event_proportion‚0.0436662957072258 9"Purity/0FRt0SRqqt2FBt/event_purity‚0.9345759153366089 :"Purity/2FRt*SRqqt1FBt/event_purity‚0.15333333611488342 @&Purity/2FRt0SRqqt1FBt/event_proportion‚0.0007672990905120969 8 Purity/2FRt2SRqqt0FBt/FRt_purity‚0.05001583695411682 8 Purity/*FRt*SRqqt0FBt/FRt_purity‚0.17296111583709717 8"Purity/0FRt*SRqqt*FBt/SRqqt_purity‚0.614582359790802 @&Purity/2FRt1SRqqt1FBt/event_proportion‚0.0008370535797439516 9"Purity/2FRt1SRqqt1FBt/SRqqt_purity‚0.7129629254341125 7 Purity/1FRt*SRqqt*FBt/FRt_purity‚0.1852761059999466 7 Purity/1FRt*SRqqt1FBt/FRt_purity‚0.5173904895782471 7 Purity/1FRt*SRqqt2FBt/FRt_purity‚0.8661565184593201 6 Purity/1FRt0SRqqt*FBt/FBt_purity‚0.996457576751709 ,&Purity/1FRt2SRqqt2FBt/event_proportion‚0 trainer/global_step‚3199 9"Purity/*FRt*SRqqt0FBt/SRqqt_purity‚0.6753975749015808 8 Purity/*FRt1SRqqt0FBt/FRt_purity‚0.13902994990348816 7 Purity/2FRt1SRqqt*FBt/FRt_purity‚0.2366381585597992 + Purity/2FRt1SRqqt1FBt/FRt_purity‚0.4375 ( Purity/2FRt1SRqqt2FBt/FRt_purity‚0.5 8 Purity/*FRt1SRqqt*FBt/FRt_purity‚0.15363118052482605 7 Purity/*FRt1SRqqt1FBt/FRt_purity‚0.4392406940460205 ) Purity/*FRt1SRqqt2FBt/FRt_purity‚0.75 8 Purity/2FRt1SRqqt0FBt/FRt_purity‚0.22363072633743286 =&Purity/*FRt0SRqqt2FBt/event_proportion‚0.0103004090487957 7 Purity/1FRt*SRqqt0FBt/FRt_purity‚0.1732444018125534 >&Purity/1FRt*SRqqt1FBt/event_proportion‚0.01643880270421505 =&Purity/*FRt*SRqqt1FBt/event_proportion‚0.0617094486951828 @&Purity/1FRt1SRqqt2FBt/event_proportion‚0.0002325148816453293 9"Purity/2FRt2SRqqt0FBt/SRqqt_purity‚0.8271432518959045 6validation_average_jet_accuracy‚0.3575437068939209 =&Purity/0FRt0SRqqt0FBt/event_proportion‚0.0462472103536129 9"Purity/1FRt1SRqqt1FBt/event_purity‚0.1943790167570114 ("Purity/1FRt2SRqqt0FBt/event_purity‚0 8 Purity/2FRt*SRqqt0FBt/FRt_purity‚0.17105895280838013 *jet/accuracy_2_of_3‚0.5335620045661926 7 Purity/*FRt*SRqqt2FBt/FBt_purity‚0.9694743156433105 8 Purity/2FRt*SRqqt*FBt/FRt_purity‚0.18697205185890198 7 Purity/2FRt*SRqqt1FBt/FRt_purity‚0.5112500190734863 ) Purity/2FRt*SRqqt2FBt/FRt_purity‚0.75 jet/accuracy_4_of_5‚0 @&Purity/*FRt2SRqqt1FBt/event_proportion‚0.0009998140158131719 9"Purity/1FRt*SRqqt2FBt/SRqqt_purity‚0.3333333432674408 ("Purity/1FRt2SRqqt1FBt/event_purity‚0 9"Purity/*FRt0SRqqt1FBt/event_purity‚0.8731292486190796 7 Purity/*FRt0SRqqt1FBt/FBt_purity‚0.9959096312522888 <&Purity/*FRt2SRqqt*FBt/event_proportion‚0.054222472012043 9"Purity/2FRt*SRqqt0FBt/SRqqt_purity‚0.7811137437820435 particle/accuracy_2_of_4‚1 'particle/f_score‚0.6283609867095947 *loss/val_total_loss‚11.318284953218669 9"Purity/0FRt*SRqqt1FBt/SRqqt_purity‚0.5017155408859253 ?&Purity/0FRt*SRqqt2FBt/event_proportion‚0.008068266324698925 *jet/accuracy_1_of_2‚0.7838008403778076 7 Purity/*FRt*SRqqt*FBt/FBt_purity‚0.9881253838539124 >&Purity/2FRt0SRqqt0FBt/event_proportion‚0.01990327425301075 ("Purity/2FRt1SRqqt0FBt/event_purity‚0 % _timestamp‚1.7844625392853994e+09 =&Purity/1FRt*SRqqt*FBt/event_proportion‚0.6297432780265808 @&Purity/2FRt*SRqqt1FBt/event_proportion‚0.0016043527284637094 +jet/accuracy_1_of_1‚0.31459107995033264 particle/accuracy_1_of_1‚1 _runtime‚ 2213.67433897 7 Purity/1FRt0SRqqt2FBt/FBt_purity‚0.9970238208770752 & Purity/1FRt2SRqqt*FBt/FBt_purity‚1 ("Purity/2FRt1SRqqt1FBt/event_purity‚0 ,&Purity/2FRt2SRqqt2FBt/event_proportion‚0 +jet/accuracy_2_of_2‚0.13187964260578156 particle/accuracy_4_of_5‚1 8 Purity/1FRt1SRqqt*FBt/FRt_purity‚0.13959576189517975 8 Purity/1FRt1SRqqt1FBt/FRt_purity‚0.43239521980285645 7 Purity/1FRt1SRqqt2FBt/FRt_purity‚0.7777777910232544 9"Purity/2FRt*SRqqt1FBt/SRqqt_purity‚0.7129629254341125 & Purity/2FRt0SRqqt1FBt/FBt_purity‚1 particle/accuracy_2_of_5‚1 9"Purity/*FRt0SRqqt*FBt/event_purity‚0.2567228674888611 9"Purity/1FRt*SRqqt1FBt/SRqqt_purity‚0.5874078273773193 9"Purity/1FRt*SRqqt2FBt/event_purity‚0.5875850319862366 8 Purity/1FRt1SRqqt0FBt/FRt_purity‚0.12508264183998108 :"Purity/*FRt2SRqqt0FBt/event_purity‚0.16884973645210266 =&Purity/0FRt1SRqqt*FBt/event_proportion‚0.2203543484210968 ,&Purity/0FRt2SRqqt2FBt/event_proportion‚0 9"Purity/1FRt0SRqqt2FBt/event_purity‚0.6441043019294739 =&Purity/2FRt0SRqqt*FBt/event_proportion‚0.0208565853536129 7 Purity/0FRt*SRqqt1FBt/FBt_purity‚0.9925335645675659 & Purity/2FRt*SRqqt*FBt/FBt_purity‚1 =&Purity/0FRt*SRqqt*FBt/event_proportion‚0.2839471697807312 =&Purity/1FRt1SRqqt*FBt/event_proportion‚0.1534598171710968 particle/accuracy_1_of_2‚1 particle/accuracy_1_of_5‚1 >&Purity/1FRt2SRqqt*FBt/event_proportion‚0.02322823740541935 :"Purity/0FRt*SRqqt2FBt/SRqqt_purity‚0.11538461595773697 9"Purity/*FRt0SRqqt2FBt/event_purity‚0.8513588905334473 9"Purity/0FRt*SRqqt0FBt/SRqqt_purity‚0.6213192939758301 =&Purity/0FRt0SRqqt*FBt/event_proportion‚0.0390159972012043 9"Purity/0FRt2SRqqt*FBt/SRqqt_purity‚0.5923957228660583 9"Purity/2FRt*SRqqt*FBt/SRqqt_purity‚0.7774693369865417 +jet/accuracy_3_of_3‚0.05287250503897667 ("Purity/2FRt1SRqqt2FBt/SRqqt_purity‚0 7 Purity/*FRt1SRqqt*FBt/FBt_purity‚0.9838072657585144 8"Purity/0FRt1SRqqt0FBt/SRqqt_purity‚0.626889705657959 , Purity/0FRt2SRqqt1FBt/FBt_purity‚0.96875 9"Purity/1FRt*SRqqt0FBt/SRqqt_purity‚0.7343286871910095 :"Purity/1FRt2SRqqt1FBt/SRqqt_purity‚0.32499998807907104 9"Purity/2FRt2SRqqt*FBt/SRqqt_purity‚0.8271432518959045 _step‚263 =&Purity/*FRt*SRqqt*FBt/event_proportion‚0.9537528157234192 7 Purity/*FRt2SRqqt1FBt/FBt_purity‚0.9814814329147339 9"Purity/0FRt2SRqqt0FBt/SRqqt_purity‚0.5981634259223938 7 Purity/1FRt0SRqqt0FBt/FRt_purity‚0.1947101652622223 ;"Purity/1FRt1SRqqt0FBt/event_purity‚0.036001384258270264 =&Purity/*FRt1SRqqt*FBt/event_proportion‚0.3866024911403656 9"Purity/0FRt*SRqqt*FBt/event_purity‚0.6449013948440552 >&Purity/0FRt2SRqqt0FBt/event_proportion‚0.02385602705180645 7 Purity/1FRt*SRqqt*FBt/FBt_purity‚0.9931647181510925 8 Purity/1FRt0SRqqt*FBt/FRt_purity‚0.20670920610427856 6 Purity/1FRt0SRqqt1FBt/FRt_purity‚0.576505184173584 7 Purity/1FRt0SRqqt2FBt/FRt_purity‚0.8765873908996582 9"Purity/1FRt1SRqqt1FBt/SRqqt_purity‚0.6047156453132629 7 Purity/0FRt0SRqqt*FBt/FBt_purity‚0.9868766069412231 9"Purity/0FRt0SRqqt1FBt/event_purity‚0.9954147338867188 7 Purity/1FRt1SRqqt2FBt/FBt_purity‚0.7777777910232544 & Purity/2FRt1SRqqt*FBt/FBt_purity‚1 *jet/accuracy_1_of_4‚0.9653623700141907 /particle/accuracy_3_of_3‚0.9994933605194092 9"Purity/*FRt1SRqqt1FBt/event_purity‚0.3418494462966919 7 Purity/0FRt0SRqqt1FBt/FBt_purity‚0.9954147338867188 & Purity/2FRt1SRqqt1FBt/FBt_purity‚1 7 Purity/1FRt*SRqqt1FBt/FBt_purity‚0.9980952143669128 ("Purity/0FRt1SRqqt2FBt/event_purity‚0 9"Purity/*FRt1SRqqt*FBt/event_purity‚0.3681221902370453 7 Purity/*FRt2SRqqt*FBt/FBt_purity‚0.9814814329147339 8&Purity/1FRt2SRqqt0FBt/event_proportion‚ 0.02294921875 +particle/specificity‚0.9021978974342346 7 Purity/*FRt1SRqqt1FBt/FBt_purity‚0.9912497401237488 , Purity/0FRt2SRqqt*FBt/FBt_purity‚0.96875 9"Purity/*FRt*SRqqt*FBt/SRqqt_purity‚0.6687334179878235 9"Purity/0FRt1SRqqt1FBt/SRqqt_purity‚0.5168287754058838 @&Purity/1FRt*SRqqt2FBt/event_proportion‚0.0027204242069274187 *jet/accuracy_2_of_4‚0.7826080322265625 +particle/sensitivity‚0.6219671964645386 9"Purity/*FRt*SRqqt1FBt/SRqqt_purity‚0.5337775349617004 :"Purity/*FRt1SRqqt0FBt/event_purity‚0.37030500173568726 9"Purity/*FRt1SRqqt2FBt/SRqqt_purity‚0.2222222238779068 8"Purity/0FRt1SRqqt*FBt/event_purity‚0.615925669670105 :"Purity/1FRt0SRqqt*FBt/event_purity‚0.20404399931430817 :"Purity/1FRt1SRqqt*FBt/event_purity‚0.04279384762048721 ?&Purity/*FRt*SRqqt2FBt/event_proportion‚0.010997953824698925 /"Purity/2FRt0SRqqt1FBt/event_purity‚0.328125 9"Purity/*FRt2SRqqt*FBt/event_purity‚0.1657545566558838 7 Purity/0FRt*SRqqt*FBt/FBt_purity‚0.9855642914772034 =&Purity/0FRt*SRqqt0FBt/event_proportion‚0.2322126179933548 9"Purity/1FRt*SRqqt*FBt/SRqqt_purity‚0.7283328175544739 9"Purity/1FRt2SRqqt*FBt/SRqqt_purity‚0.7132870554924011 & Purity/2FRt*SRqqt1FBt/FBt_purity‚1 particle/accuracy_1_of_3‚1 6 Purity/0FRt1SRqqt2FBt/FBt_purity‚0.942307710647583 9"Purity/1FRt0SRqqt1FBt/event_purity‚0.5141441822052002 particle/accuracy_2_of_3‚1 8&Purity/*FRt2SRqqt0FBt/event_proportion‚ 0.05322265625 ("Purity/0FRt0SRqqt0FBt/event_purity‚1 >&Purity/1FRt0SRqqt1FBt/event_proportion‚0.00969587080180645 ;"Purity/2FRt0SRqqt0FBt/event_purity‚0.028693709522485733 A&Purity/2FRt0SRqqt2FBt/event_proportion‚0.00018601190822664648 9"Purity/2FRt1SRqqt*FBt/SRqqt_purity‚0.7364176511764526 /particle/accuracy_3_of_4‚0.9936224818229675 jet/accuracy_2_of_5‚1 @&Purity/*FRt1SRqqt2FBt/event_proportion‚0.0006975446594879031 8"Purity/1FRt2SRqqt0FBt/SRqqt_purity‚0.718445897102356 ,&Purity/2FRt2SRqqt1FBt/event_proportion‚0 & Purity/2FRt0SRqqt*FBt/FBt_purity‚1 ?&Purity/0FRt1SRqqt1FBt/event_proportion‚0.011555989272892475 >&Purity/0FRt2SRqqt*FBt/event_proportion‚0.02457682229578495 & Purity/1FRt2SRqqt1FBt/FBt_purity‚1 @&Purity/2FRt*SRqqt2FBt/event_proportion‚0.0002092633949359879 ("Purity/2FRt2SRqqt0FBt/event_purity‚0 /particle/accuracy_4_of_4‚0.9590514302253723 jet/accuracy_5_of_5‚0 ("Purity/*FRt2SRqqt1FBt/event_purity‚0 =&Purity/1FRt1SRqqt0FBt/event_proportion‚0.1467633992433548 @&Purity/1FRt1SRqqt1FBt/event_proportion‚0.0064639137126505375 9"Purity/1FRt1SRqqt2FBt/SRqqt_purity‚0.3333333432674408 ,jet/accuracy_4_of_4‚0.013899499550461769 7 Purity/*FRt*SRqqt1FBt/FBt_purity‚0.9942828416824341 =&Purity/*FRt1SRqqt0FBt/event_proportion‚0.3670479953289032 ;"Purity/2FRt0SRqqt*FBt/event_purity‚0.042921751737594604 ?&Purity/2FRt2SRqqt0FBt/event_proportion‚0.006417410913854837 ("Purity/2FRt*SRqqt2FBt/SRqqt_purity‚0 9"Purity/*FRt*SRqqt0FBt/event_purity‚0.2624429166316986 9"Purity/*FRt*SRqqt2FBt/SRqqt_purity‚0.2222222238779068 7 Purity/*FRt0SRqqt*FBt/FBt_purity‚0.9895209074020386 ("Purity/0FRt2SRqqt1FBt/event_purity‚0 particle/accuracy_5_of_5‚0 8"Purity/*FRt2SRqqt0FBt/SRqqt_purity‚0.675209641456604 @&Purity/1FRt0SRqqt2FBt/event_proportion‚0.0024879092816263437 ("Purity/2FRt1SRqqt*FBt/event_purity‚0 +jet/accuracy_3_of_4‚0.07323213666677475 A&Purity/1FRt2SRqqt1FBt/event_proportion‚0.00027901786961592734 =&Purity/2FRt*SRqqt0FBt/event_proportion‚0.0382486991584301 <&Purity/*FRt*SRqqt0FBt/event_proportion‚0.881045401096344 =&Purity/*FRt0SRqqt1FBt/event_proportion‚0.0418526791036129 9"Purity/1FRt0SRqqt0FBt/event_purity‚0.1947101652622223 *jet/accuracy_1_of_3‚0.9143841862678528 7 Purity/*FRt0SRqqt2FBt/FBt_purity‚0.9750432372093201 =&Purity/*FRt1SRqqt1FBt/event_proportion‚0.0188569575548172 9"Purity/*FRt1SRqqt1FBt/SRqqt_purity‚0.5520342588424683 <&Purity/2FRt*SRqqt*FBt/event_proportion‚0.040062315762043 7 Purity/*FRt2SRqqt*FBt/FRt_purity‚0.0620393306016922 8 Purity/*FRt2SRqqt1FBt/FRt_purity‚0.10000000149011612 *"Purity/2FRt*SRqqt2FBt/event_purity‚0.5 epoch‚7 @&Purity/0FRt1SRqqt2FBt/event_proportion‚0.0004417782765813172 ("Purity/1FRt1SRqqt2FBt/event_purity‚0 8"Purity/2FRt1SRqqt0FBt/SRqqt_purity‚0.741261899471283 'particle/accuracy‚0.842021644115448 9"Purity/*FRt*SRqqt1FBt/event_purity‚0.6947068572044373 9"Purity/*FRt*SRqqt2FBt/event_purity‚0.7959508299827576 9"Purity/2FRt0SRqqt2FBt/event_purity‚0.5714285969734192 & Purity/2FRt0SRqqt2FBt/FBt_purity‚1 7 Purity/*FRt2SRqqt0FBt/FRt_purity‚0.0618073046207428 9"Purity/0FRt1SRqqt1FBt/event_purity‚0.4504469931125641 7 Purity/1FRt0SRqqt1FBt/FBt_purity‚0.9960317015647888 :"Purity/2FRt*SRqqt0FBt/event_purity‚0.01472106296569109 =&Purity/*FRt0SRqqt*FBt/event_proportion‚0.5129278302192688 ?&Purity/2FRt2SRqqt*FBt/event_proportion‚0.006417410913854837 particle/accuracy_2_of_2‚1 :"Purity/0FRt1SRqqt2FBt/SRqqt_purity‚0.11538461595773697 8 Purity/*FRt0SRqqt*FBt/FRt_purity‚0.20583602786064148 7 Purity/*FRt0SRqqt1FBt/FRt_purity‚0.5741528868675232 7 Purity/*FRt0SRqqt2FBt/FRt_purity‚0.8684806823730469 9"Purity/0FRt0SRqqt*FBt/event_purity‚0.9842315912246704 =&Purity/1FRt*SRqqt0FBt/event_proportion‚0.6105840802192688 8 Purity/*FRt0SRqqt0FBt/FRt_purity‚0.19320262968540192 =&Purity/0FRt1SRqqt0FBt/event_proportion‚0.2083565890789032 :"Purity/0FRt2SRqqt0FBt/event_purity‚0.38173457980155945 @&Purity/0FRt2SRqqt1FBt/event_proportion‚0.0007207961170934141 9"Purity/1FRt1SRqqt0FBt/SRqqt_purity‚0.7396588325500488 8"Purity/*FRt1SRqqt*FBt/SRqqt_purity‚0.668599009513855 7 Purity/0FRt*SRqqt2FBt/FBt_purity‚0.9645573496818542 7 Purity/0FRt1SRqqt*FBt/FBt_purity‚0.9826012849807739 8"Purity/0FRt1SRqqt0FBt/event_purity‚0.626889705657959 8"Purity/1FRt*SRqqt0FBt/event_purity‚0.149255633354187 =&Purity/1FRt0SRqqt0FBt/event_proportion‚0.4408714771270752 jet/accuracy_1_of_5‚1 ("Purity/*FRt1SRqqt2FBt/event_purity‚0 9"Purity/0FRt*SRqqt0FBt/event_purity‚0.6012110710144043 9"Purity/0FRt1SRqqt*FBt/SRqqt_purity‚0.6198163628578186 /"Purity/0FRt2SRqqt1FBt/SRqqt_purity‚0.421875 =&Purity/1FRt0SRqqt*FBt/event_proportion‚0.4530552327632904 9"Purity/1FRt1SRqqt*FBt/SRqqt_purity‚0.7331354022026062 & Purity/1FRt1SRqqt1FBt/FBt_purity‚1 8 Purity/2FRt0SRqqt*FBt/FRt_purity‚0.19736146926879883 =&Purity/*FRt0SRqqt0FBt/event_proportion‚0.4607747495174408 :"Purity/*FRt*SRqqt*FBt/event_purity‚0.29666459560394287 ?&Purity/0FRt0SRqqt2FBt/event_proportion‚0.007626487873494625 =&Purity/2FRt1SRqqt*FBt/event_proportion‚0.0127883180975914 7 Purity/2FRt0SRqqt1FBt/FRt_purity‚0.5989583730697632 7 Purity/2FRt0SRqqt2FBt/FRt_purity‚0.7857142686843872 9"Purity/*FRt2SRqqt*FBt/SRqqt_purity‚0.6696025133132935 :"Purity/1FRt*SRqqt*FBt/event_purity‚0.15721580386161804 ("Purity/1FRt2SRqqt*FBt/event_purity‚0 particle/accuracy_3_of_5‚1 :"Purity/*FRt2SRqqt1FBt/SRqqt_purity‚0.37962964177131653 8 Purity/2FRt0SRqqt0FBt/FRt_purity‚0.17726553976535797 7 Purity/*FRt1SRqqt2FBt/FBt_purity‚0.8611111044883728 jet/accuracy_3_of_5‚1 :"Purity/*FRt0SRqqt0FBt/event_purity‚0.18750058114528656 8 Purity/1FRt2SRqqt0FBt/FRt_purity‚0.06659070402383804Ð=;´ü õ{ /particle/accuracy_3_of_3‚0.9994933605194092 9"Purity/*FRt1SRqqt1FBt/event_purity‚0.3418494462966919 7 Purity/0FRt0SRqqt1FBt/FBt_purity‚0.9954147338867188 & Purity/2FRt1SRqqt1FBt/FBt_purity‚1 7 Purity/1FRt*SRqqt1FBt/FBt_purity‚0.9980952143669128 ("Purity/0FRt1SRqqt2FBt/event_purity‚0 9"Purity/*FRt1SRqqt*FBt/event_purity‚0.3681221902370453 7 Purity/*FRt2SRqqt*FBt/FBt_purity‚0.9814814329147339 8&Purity/1FRt2SRqqt0FBt/event_proportion‚ 0.02294921875 +particle/specificity‚0.9021978974342346 7 Purity/*FRt1SRqqt1FBt/FBt_purity‚0.9912497401237488 , Purity/0FRt2SRqqt*FBt/FBt_purity‚0.96875 9"Purity/*FRt*SRqqt*FBt/SRqqt_purity‚0.6687334179878235 9"Purity/0FRt1SRqqt1FBt/SRqqt_purity‚0.5168287754058838 @&Purity/1FRt*SRqqt2FBt/event_proportion‚0.0027204242069274187 *jet/accuracy_2_of_4‚0.7826080322265625 +particle/sensitivity‚0.6219671964645386 9"Purity/*FRt*SRqqt1FBt/SRqqt_purity‚0.5337775349617004 :"Purity/*FRt1SRqqt0FBt/event_purity‚0.37030500173568726 9"Purity/*FRt1SRqqt2FBt/SRqqt_purity‚0.2222222238779068 8"Purity/0FRt1SRqqt*FBt/event_purity‚0.615925669670105 :"Purity/1FRt0SRqqt*FBt/event_purity‚0.20404399931430817 :"Purity/1FRt1SRqqt*FBt/event_purity‚0.04279384762048721 ?&Purity/*FRt*SRqqt2FBt/event_proportion‚0.010997953824698925 /"Purity/2FRt0SRqqt1FBt/event_purity‚0.328125 9"Purity/*FRt2SRqqt*FBt/event_purity‚0.1657545566558838 7 Purity/0FRt*SRqqt*FBt/FBt_purity‚0.9855642914772034 =&Purity/0FRt*SRqqt0FBt/event_proportion‚0.2322126179933548 9"Purity/1FRt*SRqqt*FBt/SRqqt_purity‚0.7283328175544739 9"Purity/1FRt2SRqqt*FBt/SRqqt_purity‚0.7132870554924011 & Purity/2FRt*SRqqt1FBt/FBt_purity‚1 ñ•ñqG7particle/accuracy_1_of_3‚1 6 Purity/0FRt1SRqqt2FBt/FBt_purity‚0.942307710647583 9"Purity/1FRt0SRqqt1FBt/event_purity‚0.5141441822052002 particle/accuracy_2_of_3‚1 8&Purity/*FRt2SRqqt0FBt/event_proportion‚ 0.05322265625 ("Purity/0FRt0SRqqt0FBt/event_purity‚1 >&Purity/1FRt0SRqqt1FBt/event_proportion‚0.00969587080180645 ;"Purity/2FRt0SRqqt0FBt/event_purity‚0.028693709522485733 A&Purity/2FRt0SRqqt2FBt/event_proportion‚0.00018601190822664648 9"Purity/2FRt1SRqqt*FBt/SRqqt_purity‚0.7364176511764526 /particle/accuracy_3_of_4‚0.9936224818229675 jet/accuracy_2_of_5‚1 @&Purity/*FRt1SRqqt2FBt/event_proportion‚0.0006975446594879031 8"Purity/1FRt2SRqqt0FBt/SRqqt_purity‚0.718445897102356 ,&Purity/2FRt2SRqqt1FBt/event_proportion‚0 & Purity/2FRt0SRqqt*FBt/FBt_purity‚1 ?&Purity/0FRt1SRqqt1FBt/event_proportion‚0.011555989272892475 >&Purity/0FRt2SRqqt*FBt/event_proportion‚0.02457682229578495 & Purity/1FRt2SRqqt1FBt/FBt_purity‚1 @&Purity/2FRt*SRqqt2FBt/event_proportion‚0.0002092633949359879 ("Purity/2FRt2SRqqt0FBt/event_purity‚0 /particle/accuracy_4_of_4‚0.9590514302253723 jet/accuracy_5_of_5‚0 ("Purity/*FRt2SRqqt1FBt/event_purity‚0 =&Purity/1FRt1SRqqt0FBt/event_proportion‚0.1467633992433548 @&Purity/1FRt1SRqqt1FBt/event_proportion‚0.0064639137126505375 9"Purity/1FRt1SRqqt2FBt/SRqqt_purity‚0.3333333432674408 ,jet/accuracy_4_of_4‚0.013899499550461769 7 Purity/*FRt*SRqqt1FBt/FBt_purity‚0.9942828416824341 =&Purity/*FRt1SRqqt0FBt/event_proportion‚0.3670479953289032 ;"Purity/2FRt0SRqqt*FBt/event_purity‚0.042921751737594604 ?&Purity/2FRt2SRqqt0FBt/event_proportion‚0.006417410913854837 ("Purity/2FRt*SRqqt2FBt/SRqqt_purity‚0 9"Purity/*FRt*SRqqt0FBt/event_purity‚0.2624429166316986 9"Purity/*FRt*SRqqt2FBt/SRqqt_purity‚0.2222222238779068 7 Purity/*FRt0SRqqt*FBt/FBt_purity‚0.9895209074020386 ("Purity/0FRt2SRqqt1FBt/event_purity‚0 particle/accuracy_5_of_5‚0 8"Purity/*FRt2SRqqt0FBt/SRqqt_purity‚0.675209641456604 @&Purity/1FRt0SRqqt2FBt/event_proportion‚0.0024879092816263437 ("Purity/2FRt1SRqqt*FBt/event_purity‚0 +jet/accuracy_3_of_4‚0.07323213666677475 A&Purity/1FRt2SRqqt1FBt/event_proportion‚0.00027901786961592734 =&Purity/2FRt*SRqqt0FBt/event_proportion‚0.0382486991584301 <&Purity/*FRt*SRqqt0FBt/event_proportion‚0.881045401096344 =&Purity/*FRt0SRqqt1FBt/event_proportion‚0.0418526791036129 9"Purity/1FRt0SRqqt0FBt/event_purity‚0.1947101652622223 *jet/accuracy_1_of_3‚0.9143841862678528 7 Purity/*FRt0SRqqt2FBt/FBt_purity‚0.9750432372093201 =&Purity/*FRt1SRqqt1FBt/event_proportion‚0.0188569575548172 9"Purity/*FRt1SRqqt1FBt/SRqqt_purity‚0.5520342588424683 <&Purity/2FRt*SRqqt*FBt/event_proportion‚0.040062315762043 7 Purity/*FRt2SRqqt*FBt/FRt_purity‚0.0620393306016922 8 Purity/*FRt2SRqqt1FBt/FRt_purity‚0.10000000149011612 *"Purity/2FRt*SRqqt2FBt/event_purity‚0.5 epoch‚7 @&Purity/0FRt1SRqqt2FBt/event_proportion‚0.0004417782765813172 ("Purity/1FRt1SRqqt2FBt/event_purity‚0 8"Purity/2FRt1SRqqt0FBt/SRqqt_purity‚0.741261899471283 'particle/accuracy‚0.842021644115448 9"Purity/*FRt*SRqqt1FBt/event_purity‚0.6947068572044373 9"Purity/*FRt*SRqqt2FBt/event_purity‚0.7959508299827576 9"Purity/2FRt0SRqqt2FBt/event_purity‚0.5714285969734192 & Purity/2FRt0SRqqt2FBt/FBt_purity‚1 7 Purity/*FRt2SRqqt0FBt/FRt_purity‚0.0618073046207428 9"Purity/0FRt1SRqqt1FBt/event_purity‚0.4504469931125641 7 Purity/1FRt0SRqqt1FBt/FBt_purity‚0.9960317015647888 :"Purity/2FRt*SRqqt0FBt/event_purity‚0.01472106296569109 =&Purity/*FRt0SRqqt*FBt/event_proportion‚0.5129278302192688 ?&Purity/2FRt2SRqqt*FBt/event_proportion‚0.006417410913854837 particle/accuracy_2_of_2‚1 :"Purity/0FRt1SRqqt2FBt/SRqqt_purity‚0.11538461595773697 8 Purity/*FRt0SRqqt*FBt/FRt_purity‚0.20583602786064148 7 Purity/*FRt0SRqqt1FBt/FRt_purity‚0.5741528868675232 7 Purity/*FRt0SRqqt2FBt/FRt_purity‚0.8684806823730469 9"Purity/0FRt0SRqqt*FBt/event_purity‚0.9842315912246704 =&Purity/1FRt*SRqqt0FBt/event_proportion‚0.6105840802192688 8 Purity/*FRt0SRqqt0FBt/FRt_purity‚0.19320262968540192 =&Purity/0FRt1SRqqt0FBt/event_proportion‚0.2083565890789032 :"Purity/0FRt2SRqqt0FBt/event_purity‚0.38173457980155945 @&Purity/0FRt2SRqqt1FBt/event_proportion‚0.0007207961170934141 9"Purity/1FRt1SRqqt0FBt/SRqqt_purity‚0.7396588325500488 8"Purity/*FRt1SRqqt*FBt/SRqqt_purity‚0.668599009513855 7 Purity/0FRt*SRqqt2FBt/FBt_purity‚0.9645573496818542 7 Purity/0FRt1SRqqt*FBt/FBt_purity‚0.9826012849807739 8"Purity/0FRt1SRqqt0FBt/event_purity‚0.626889705657959 8"Purity/1FRt*SRqqt0FBt/event_purity‚0.149255633354187 =&Purity/1FRt0SRqqt0FBt/event_proportion‚0.4408714771270752 jet/accuracy_1_of_5‚1 ("Purity/*FRt1SRqqt2FBt/event_purity‚0 9"Purity/0FRt*SRqqt0FBt/event_purity‚0.6012110710144043 9"Purity/0FRt1SRqqt*FBt/SRqqt_purity‚0.6198163628578186 /"Purity/0FRt2SRqqt1FBt/SRqqt_purity‚0.421875 =&Purity/1FRt0SRqqt*FBt/event_proportion‚0.4530552327632904 9"Purity/1FRt1SRqqt*FBt/SRqqt_purity‚0.7331354022026062 & Purity/1FRt1SRqqt1FBt/FBt_purity‚1 8 Purity/2FRt0SRqqt*FBt/FRt_purity‚0.19736146926879883 =&Purity/*FRt0SRqqt0FBt/event_proportion‚0.4607747495174408 :"Purity/*FRt*SRqqt*FBt/event_purity‚0.29666459560394287 ?&Purity/0FRt0SRqqt2FBt/event_proportion‚0.007626487873494625 =&Purity/2FRt1SRqqt*FBt/event_proportion‚0.0127883180975914 7 Purity/2FRt0SRqqt1FBt/FRt_purity‚0.5989583730697632 7 Purity/2FRt0SRqqt2FBt/FRt_purity‚0.7857142686843872 9"Purity/*FRt2SRqqt*FBt/SRqqt_purity‚0.6696025133132935 :"Purity/1FRt*SRqqt*FBt/event_purity‚0.15721580386161804 ("Purity/1FRt2SRqqt*FBt/event_purity‚0 particle/accuracy_3_of_5‚1 :"Purity/*FRt2SRqqt1FBt/SRqqt_purity‚0.37962964177131653 8 Purity/2FRt0SRqqt0FBt/FRt_purity‚0.17726553976535797 7 Purity/*FRt1SRqqt2FBt/FBt_purity‚0.8611111044883728 jet/accuracy_3_of_5‚1 :"Purity/*FRt0SRqqt0FBt/event_purity‚0.18750058114528656 8 Purity/1FRt2SRqqt0FBt/FRt_purity‚0.06659070402383804 9"Purity/*FRt1SRqqt0FBt/SRqqt_purity‚0.6755766868591309 9"Purity/0FRt*SRqqt1FBt/event_purity‚0.8309464454650879 :"Purity/0FRt2SRqqt*FBt/event_purity‚0.37108752131462097 7 Purity/1FRt*SRqqt2FBt/FBt_purity‚0.9815192222595215 7 Purity/1FRt2SRqqt*FBt/FRt_purity‚0.0668833926320076 8 Purity/1FRt2SRqqt1FBt/FRt_purity‚0.10000000149011612 ;"Purity/2FRt*SRqqt*FBt/event_purity‚0.022109752520918846 particle/accuracy_1_of_4‚1 9"Purity/0FRt*SRqqt2FBt/event_purity‚0.8800085783004761 =&Purity/0FRt0SRqqt1FBt/event_proportion‚0.0313895083963871 7 Purity/0FRt0SRqqt2FBt/FBt_purity‚0.9672878980636597 7 Purity/1FRt1SRqqt*FBt/FBt_purity‚0.9872215986251831 ?&Purity/2FRt1SRqqt0FBt/event_proportion‚0.011928013525903225 @&Purity/2FRt1SRqqt2FBt/event_proportion‚2.325148852833081e-05 ("Purity/2FRt1SRqqt2FBt/event_purity‚0 & Purity/2FRt1SRqqt2FBt/FBt_purity‚1 8 Purity/*FRt*SRqqt*FBt/FRt_purity‚0.18540947139263153 7 Purity/*FRt*SRqqt1FBt/FRt_purity‚0.5140624642372131 7 Purity/*FRt*SRqqt2FBt/FRt_purity‚0.8554421663284302 7 Purity/0FRt1SRqqt1FBt/FBt_purity‚0.9862236976623535 & Purity/2FRt*SRqqt2FBt/FBt_purity‚1 ,&Purity/*FRt2SRqqt2FBt/event_proportion‚0 9"Purity/1FRt*SRqqt1FBt/event_purity‚0.3836148679256439 ("Purity/2FRt2SRqqt*FBt/event_purity‚0 8 Purity/2FRt2SRqqt*FBt/FRt_purity‚0.05001583695411682 =&Purity/0FRt*SRqqt1FBt/event_proportion‚0.0436662957072258 9"Purity/0FRt0SRqqt2FBt/event_purity‚0.9345759153366089 :"Purity/2FRt*SRqqt1FBt/event_purity‚0.15333333611488342 @&Purity/2FRt0SRqqt1FBt/event_proportion‚0.0007672990905120969 8 Purity/2FRt2SRqqt0FBt/FRt_purity‚0.05001583695411682 8 Purity/*FRt*SRqqt0FBt/FRt_purity‚0.17296111583709717 8"Purity/0FRt*SRqqt*FBt/SRqqt_purity‚0.614582359790802 @&Purity/2FRt1SRqqt1FBt/event_proportion‚0.0008370535797439516 9"Purity/2FRt1SRqqt1FBt/SRqqt_purity‚0.7129629254341125 7 Purity/1FRt*SRqqt*FBt/FRt_purity‚0.1852761059999466 7 Purity/1FRt*SRqqt1FBt/FRt_purity‚0.5173904895782471 7 Purity/1FRt*SRqqt2FBt/FRt_purity‚0.8661565184593201 6 Purity/1FRt0SRqqt*FBt/FBt_purity‚0.996457576751709 ,&Purity/1FRt2SRqqt2FBt/event_proportion‚0 trainer/global_step‚3199 9"Purity/*FRt*SRqqt0FBt/SRqqt_purity‚0.6753975749015808 8 Purity/*FRt1SRqqt0FBt/FRt_purity‚0.13902994990348816 7 Purity/2FRt1SRqqt*FBt/FRt_purity‚0.2366381585597992 + Purity/2FRt1SRqqt1FBt/FRt_purity‚0.4375 ( Purity/2FRt1SRqqt2FBt/FRt_purity‚0.5 8 Purity/*FRt1SRqqt*FBt/FRt_purity‚0.15363118052482605 7 Purity/*FRt1SRqqt1FBt/FRt_purity‚0.4392406940460205 ) Purity/*FRt1SRqqt2FBt/FRt_purity‚0.75 8 Purity/2FRt1SRqqt0FBt/FRt_purity‚0.22363072633743286 =&Purity/*FRt0SRqqt2FBt/event_proportion‚0.0103004090487957 7 Purity/1FRt*SRqqt0FBt/FRt_purity‚0.1732444018125534 >&Purity/1FRt*SRqqt1FBt/event_proportion‚0.01643880270421505 =&Purity/*FRt*SRqqt1FBt/event_proportion‚0.0617094486951828 @&Purity/1FRt1SRqqt2FBt/event_proportion‚0.0002325148816453293 9"Purity/2FRt2SRqqt0FBt/SRqqt_purity‚0.8271432518959045 6validation_average_jet_accuracy‚0.3575437068939209 =&Purity/0FRt0SRqqt0FBt/event_proportion‚0.0462472103536129 9"Purity/1FRt1SRqqt1FBt/event_purity‚0.1943790167570114 ("Purity/1FRt2SRqqt0FBt/event_purity‚0 8 Purity/2FRt*SRqqt0FBt/FRt_purity‚0.17105895280838013 *jet/accuracy_2_of_3‚0.5335620045661926 7 Purity/*FRt*SRqqt2FBt/FBt_purity‚0.9694743156433105 8 Purity/2FRt*SRqqt*FBt/FRt_purity‚0.18697205185890198 7 Purity/2FRt*SRqqt1FBt/FRt_purity‚0.5112500190734863 ) Purity/2FRt*SRqqt2FBt/FRt_purity‚0.75 jet/accuracy_4_of_5‚0 @&Purity/*FRt2SRqqt1FBt/event_proportion‚0.0009998140158131719 9"Purity/1FRt*SRqqt2FBt/SRqqt_purity‚0.3333333432674408 ("Purity/1FRt2SRqqt1FBt/event_purity‚0 9"Purity/*FRt0SRqqt1FBt/event_purity‚0.8731292486190796 7 Purity/*FRt0SRqqt1FBt/FBt_purity‚0.9959096312522888 <&Purity/*FRt2SRqqt*FBt/event_proportion‚0.054222472012043 9"Purity/2FRt*SRqqt0FBt/SRqqt_purity‚0.7811137437820435 particle/accuracy_2_of_4‚1 'particle/f_score‚0.6283609867095947 *loss/val_total_loss‚11.318284953218669 9"Purity/0FRt*SRqqt1FBt/SRqqt_purity‚0.5017155408859253 ?&Purity/0FRt*SRqqt2FBt/event_proportion‚0.008068266324698925 *jet/accuracy_1_of_2‚0.7838008403778076 7 Purity/*FRt*SRqqt*FBt/FBt_purity‚0.9881253838539124 >&Purity/2FRt0SRqqt0FBt/event_proportion‚0.01990327425301075 ("Purity/2FRt1SRqqt0FBt/event_purity‚0 % _timestamp‚1.7844625392853994e+09 =&Purity/1FRt*SRqqt*FBt/event_proportion‚0.6297432780265808 @&Purity/2FRt*SRqqt1FBt/event_proportion‚0.0016043527284637094 +jet/accuracy_1_of_1‚0.31459107995033264 particle/accuracy_1_of_1‚1 _runtime‚ 2213.67433897 7 Purity/1FRt0SRqqt2FBt/FBt_purity‚0.9970238208770752 & Purity/1FRt2SRqqt*FBt/FBt_purity‚1 ("Purity/2FRt1SRqqt1FBt/event_purity‚0 ,&Purity/2FRt2SRqqt2FBt/event_proportion‚0 +jet/accuracy_2_of_2‚0.13187964260578156 particle/accuracy_4_of_5‚1 8 Purity/1FRt1SRqqt*FBt/FRt_purity‚0.13959576189517975 8 Purity/1FRt1SRqqt1FBt/FRt_purity‚0.43239521980285645 7 Purity/1FRt1SRqqt2FBt/FRt_purity‚0.7777777910232544 9"Purity/2FRt*SRqqt1FBt/SRqqt_purity‚0.7129629254341125 & Purity/2FRt0SRqqt1FBt/FBt_purity‚1 particle/accuracy_2_of_5‚1 9"Purity/*FRt0SRqqt*FBt/event_purity‚0.2567228674888611 9"Purity/1FRt*SRqqt1FBt/SRqqt_purity‚0.5874078273773193 9"Purity/1FRt*SRqqt2FBt/event_purity‚0.5875850319862366 8 Purity/1FRt1SRqqt0FBt/FRt_purity‚0.12508264183998108 :"Purity/*FRt2SRqqt0FBt/event_purity‚0.16884973645210266 =&Purity/0FRt1SRqqt*FBt/event_proportion‚0.2203543484210968 ,&Purity/0FRt2SRqqt2FBt/event_proportion‚0 9"Purity/1FRt0SRqqt2FBt/event_purity‚0.6441043019294739 =&Purity/2FRt0SRqqt*FBt/event_proportion‚0.0208565853536129 7 Purity/0FRt*SRqqt1FBt/FBt_purity‚0.9925335645675659 & Purity/2FRt*SRqqt*FBt/FBt_purity‚1 =&Purity/0FRt*SRqqt*FBt/event_proportion‚0.2839471697807312 =&Purity/1FRt1SRqqt*FBt/event_proportion‚0.1534598171710968 particle/accuracy_1_of_2‚1 particle/accuracy_1_of_5‚1 >&Purity/1FRt2SRqqt*FBt/event_proportion‚0.02322823740541935 :"Purity/0FRt*SRqqt2FBt/SRqqt_purity‚0.11538461595773697 9"Purity/*FRt0SRqqt2FBt/event_purity‚0.8513588905334473 9"Purity/0FRt*SRqqt0FBt/SRqqt_purity‚0.6213192939758301 =&Purity/0FRt0SRqqt*FBt/event_proportion‚0.0390159972012043 9"Purity/0FRt2SRqqt*FBt/SRqqt_purity‚0.5923957228660583 9"Purity/2FRt*SRqqt*FBt/SRqqt_purity‚0.7774693369865417 +jet/accuracy_3_of_3‚0.05287250503897667 ("Purity/2FRt1SRqqt2FBt/SRqqt_purity‚0 7 Purity/*FRt1SRqqt*FBt/FBt_purity‚0.9838072657585144 8"Purity/0FRt1SRqqt0FBt/SRqqt_purity‚0.626889705657959 , Purity/0FRt2SRqqt1FBt/FBt_purity‚0.96875 9"Purity/1FRt*SRqqt0FBt/SRqqt_purity‚0.7343286871910095 :"Purity/1FRt2SRqqt1FBt/SRqqt_purity‚0.32499998807907104 9"Purity/2FRt2SRqqt*FBt/SRqqt_purity‚0.8271432518959045 _step‚263 =&Purity/*FRt*SRqqt*FBt/event_proportion‚0.9537528157234192 7 Purity/*FRt2SRqqt1FBt/FBt_purity‚0.9814814329147339 9"Purity/0FRt2SRqqt0FBt/SRqqt_purity‚0.5981634259223938 7 Purity/1FRt0SRqqt0FBt/FRt_purity‚0.1947101652622223 ;"Purity/1FRt1SRqqt0FBt/event_purity‚0.036001384258270264 =&Purity/*FRt1SRqqt*FBt/event_proportion‚0.3866024911403656 9"Purity/0FRt*SRqqt*FBt/event_purity‚0.6449013948440552 >&Purity/0FRt2SRqqt0FBt/event_proportion‚0.02385602705180645 7 Purity/1FRt*SRqqt*FBt/FBt_purity‚0.9931647181510925 8 Purity/1FRt0SRqqt*FBt/FRt_purity‚0.20670920610427856 6 Purity/1FRt0SRqqt1FBt/FRt_purity‚0.576505184173584 7 Purity/1FRt0SRqqt2FBt/FRt_purity‚0.8765873908996582 9"Purity/1FRt1SRqqt1FBt/SRqqt_purity‚0.6047156453132629 7 Purity/0FRt0SRqqt*FBt/FBt_purity‚0.9868766069412231 9"Purity/0FRt0SRqqt1FBt/event_purity‚0.9954147338867188 7 Purity/1FRt1SRqqt2FBt/FBt_purity‚0.7777777910232544 & Purity/2FRt1SRqqt*FBt/FBt_purity‚1 *jet/accuracy_1_of_4‚0.9653623700141907‡I`’rý ‚(:ç ÑùòÒ…Õ“¥ disk.dm-0.in‚ 1982.2109375# memory_percent‚99.1342306137085% disk.nvme0n1.out‚25802.2744140625% disk.nvme1n1.out‚14867.5302734375 disk.dm-1.in‚0# proc.memory.rssMB‚ 3017.04296875( disk./.usagePercent‚76.1522592191412" disk.nvme0c0n1.in‚ 1852.4453125 disk.nvme1c1n1.in‚ 129.765625' disk.nvme1c1n1.out‚14867.5302734375 disk.nvme1n1.in‚ 129.765625" disk.dm-0.out‚39032.3271484375 disk.rbd0.out‚0) proc.memory.percent‚18.41456890106201 network.sent‚ 4.736617e+06 network.recv‚607513' disk.nvme0c0n1.out‚25802.2744140625! disk.dm-1.out‚1517.8564453125 disk.rbd0.in‚0( proc.memory.availableMB‚ 141.84765625 cpu‚25.849751353283548 proc.cpu.threads‚17 disk.md0.out‚ 55.61328125% disk./.usageGB‚1169.1275596618652 disk.nvme0n1.in‚ 1852.4453125 disk.md0.in‚0¡þ­oœþ ‚(:‘ ÑùòÒÃÄÉ¥ gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚48.0 gpu.0.powerWatts‚113.165' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚37.72166666666667 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Ÿƒ~^xÿ ‚(:í àùòÒ‘ƒÉ¤ disk.dm-0.in‚ 1982.2109375 disk.dm-1.in‚0 disk.rbd0.in‚0 disk.rbd0.out‚0" disk.nvme0c0n1.in‚ 1852.4453125( disk.nvme0c0n1.out‚25806.50634765625 disk.nvme0n1.in‚ 1852.4453125 disk.nvme1n1.in‚ 129.765625& disk.nvme1n1.out‚14932.67236328125 disk.md0.in‚0$ memory_percent‚99.13420677185059' proc.memory.availableMB‚ 141.8515625) proc.memory.percent‚18.41456890106201 cpu‚25.847876854220633 disk.nvme1c1n1.in‚ 129.765625 disk.md0.out‚ 55.8515625# proc.memory.rssMB‚ 3017.04296875 proc.cpu.threads‚17 network.sent‚ 4.782846e+06 network.recv‚611673) disk./.usagePercent‚76.15225549203102% disk./.usageGB‚1169.1275024414062# disk.dm-0.out‚39037.12548828125" disk.dm-1.out‚1581.89306640625& disk.nvme0n1.out‚25806.50634765625( disk.nvme1c1n1.out‚14932.67236328125Aƒ¬“€ ‚(:ˆ àùòÒ¦üü¤ gpu.0.gpu‚92.0 gpu.0.memory‚20, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚48.0 gpu.0.powerWatts‚223.008' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚74.336 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0)w2 - _wandbruntime‚2245  _runtime‚2245ŒUŽÈ‚  $ _timestamp‚1.784462571241331e+09 _runtime‚2245.629740805 _step‚264 ' lr-AdamW/pg1‚0.00013210873319652135 ' lr-AdamW/pg2‚0.00013210873319652135 trainer/global_step‚3249Õ¬%̓ Ç ' lr-AdamW/pg1‚0.00013210873319652135 ' lr-AdamW/pg2‚0.00013210873319652135 trainer/global_step‚3249 $ _timestamp‚1.784462571241331e+09 _runtime‚2245.629740805 _step‚264ˆµ ¤¨2„ - _wandbruntime‚2245  _runtime‚2245oiƒ à'… ÚO IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 % _timestamp‚1.7844625712422128e+09 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚254909431526912 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 215164928 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚254909325827584 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚25772623 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚188919889840640 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚184 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2761603716608 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚7372400 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚191681278392320 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚254909431526912 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚7372584 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 _runtime‚2245.633751295 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚249250364434448 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚32702710 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚252227483053568 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚17299337 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚188919795730944 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚32692567 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚254909325827584 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2681842774016 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 121055232 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚252227500092928 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚249250381473808 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2674575524418 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 !ez† ßO G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚254909325827584 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚25772623 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚188919889840640 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚184 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2761603716608 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚7372400 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚191681278392320 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚254909431526912 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚7372584 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 _runtime‚2245.633751295 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚249250364434448 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚32702710 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚252227483053568 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚17299337 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚188919795730944 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚32692567 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚254909325827584 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2681842774016 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 121055232 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚252227500092928 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚249250381473808 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2674575524418 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 % _timestamp‚1.7844625712422128e+09 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚254909431526912 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 215164928 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 IDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚252302649759232 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚17304397 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚58482637 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚32712544 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚24679073 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚32702387 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚252302649759232 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚25780252 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚24678898 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚188976027568640 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2675359019130 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚191738223691264 # _timestamp‚1.78446257164566e+09 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚58492796 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚17304393 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚249324671999200 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚25780252 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 219352064 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 _runtime‚2246.034202748 _step‚266 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚254985261209600 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0GÜã7í&‰ çM REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚32712544 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚24678898 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚188976027568640 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2675359019130 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚191738223691264 # _timestamp‚1.78446257164566e+09 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚58492796 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚17304393 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚249324671999200 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚25780252 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 219352064 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 _runtime‚2246.034202748 _step‚266 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚254985261209600 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚252302649759232 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚17304397 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚58482637 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚32712544 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚24679073 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚32702387 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚252302649759232 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚25780252 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚25032771 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚17552137 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2802265763840 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚258706159083008 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚174 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes. ¯]u$small_pool.allocated‚ 2713814055226 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚59349635 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 206776320 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 112666624 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2721281374208 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚258706264782336 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚170 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚59339490 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 $ _timestamp‚1.784462603522232e+09 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚33185419 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚17552141 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚255678617972848 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚7480634 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2713727268800 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚7480804 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚255984966368768 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚26154071 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚26154071 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚191726682130944 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚255984983408128 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚194529042004480 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚194528835228160 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚33195562 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚255984983408128 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚255984966368768 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚258706264782336 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 150006272 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚252964890704048 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2721192714240 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚25032945 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _runtime‚2277.910616723 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚255678721798634 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚26154073 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130»¶gÆä'– ÞO RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚255984966368768 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚258706264782336 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 150006272 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚252964890704048 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2721192714240 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚25032945 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _runtime‚2277.910616723 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚255678721798634 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚26154073 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚33185419 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚25032771 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚17552137 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2802265763840 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚258706159083008 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚174 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2713814055226 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚59349635 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 206776320 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 112666624 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2721281374208 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚258706264782336 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚170 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚59339490 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 $ _timestamp‚1.784462603522232e+09 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚33185419 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚17552141 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚255678617972848 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚7480634 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2713727268800 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚7480804 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚255984966368768 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚26154071 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚26154071 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚191726682130944 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚255984983408128 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚194529042004480 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚194528835228160 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚33195562 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚255984983408128 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248Œ Qâ2— - _wandbruntime‚2278  _runtime‚2278Mÿµé&˜ ãM NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚26161702 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2714597549938 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚33195239 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚258782200171520 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2722067097088 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2722067097088 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚255753795818738 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚59367098 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚59356939 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚33205396 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚17557193 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚7482734 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 150006272 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚253039181229440 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 $ _timestamp‚1.784462603927377e+09 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚25039927 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2721978429952 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚256060116035072 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚194585998404608 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚191782819858944 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚59367098 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚33205396 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2714510763400 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚26161700 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 217254912 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚253039198268800 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚26161702 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚194585781149696 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚191782913968640 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2802961290752 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚258782200171520 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚25040111 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚180 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚184 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 _step‚270 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚26161700 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚33195239 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0òäxŒº™ èM UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚7482914 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2722067097088 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚255753795818738 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚59367098 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚59356939 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚33205396 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚17557193 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚7482734 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 150006272 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚253039181229440 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 $ _timestamp‚1.784462603927377e+09 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚25039927 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2721978429952 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚256060116035072 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚194585998404608 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚191782819858944 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚59367098 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚33205396 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2714510763400 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚26161700 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 217254912 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚253039198268800 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚26161702 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚194585781149696 cODeviceStatsMonitor.on_traiÑ|>É4n_batch_end/inactive_split_bytes.large_pool.allocated‚191782913968640 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2802961290752 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚258782200171520 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚25040111 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚180 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚184 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 _step‚270 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚26161700 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚33195239 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚26161702 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2714597549938 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚33195239 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚258782200171520 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2722067097088 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 PDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚262502992338432 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚262503098037760 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚259742466723328 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚17804937 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2760631314432 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚259742466723328 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 217262080 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2842879120384 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2842755968000 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 _runtime‚2310.224415593 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚33688414 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚181 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚7589256 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚197376541761024 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚26535523 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚256679434013008 _step‚273 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚7589079 trainer/global_step‚3349 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚60213792 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚60223937 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2760542654464 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚259742449683968 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2753052586034 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚26535523 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚25394016 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚17804941 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145Ϩíå'¨ ßO TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚259742449683968 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2753052586034 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚26535523 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚25394016 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚17804941 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚194533662640640 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚262502992338432 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚262503098037760 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚259742466723328 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚17804937 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2760631314432 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚259742466723328 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 217262080 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2842879120384 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2842755968000 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 _runtime‚2310.224415593 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚33688414 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚181 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚7589256 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚197376541761024 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚26535523 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚256679434013008 _step‚273 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚7589079 trainer/global_step‚3349 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚60213792 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚60223937 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2760542654464 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0‘‹¸2© - _wandbruntime‚2310  _runtime‚2310Ò6×éª äM JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚26543150 _runtime‚2310.941168484 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚33698248 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2843567405568 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚17809993 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚262579033426944 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚33688091 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚60231241 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚26543150 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚177 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2761417037312 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 % _timestamp‚1.7844626365526197e+09 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2761417037312 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚33688091 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚259817616389632 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚60241400 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚17809997 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚7591427 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚7591250 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚25401243 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚256753724538400 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚26543152 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2843688453632 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚259507560619146 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚33698248 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚262579033426944 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚259817599350272 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚194589706258944 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 121048064 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚256753707499040 _step‚274 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚26543152†Ï]Fï&« éM UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚33698248 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2843567405568 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚17809993 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚262579033426944 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚33688091 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚60231241 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚26543150 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚177 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2761417037312 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 % _timestamp‚1.7844626365526197e+09 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2761417037312 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚33688091 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚259817616389632 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚60241400 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚17809997 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚7591427 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚7591250 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚25401243 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚256753724538400 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚26543152 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2843688453632 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚259507560619146 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚33698248 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚262579033426944 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚259817599350272 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚194589706258944 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 121048064 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚256753707499040 _step‚274 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚26543152 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚26543150 _runtime‚2310.941168484’ƒ^2¬ - _wandbruntime‚2310  _runtime‚2310,<ê­ € /loss/FRt2/detection_loss‚1.4190087409394572 1loss/SRqqt2/detection_loss‚0.7163152899571726 /loss/FRt2/assignment_loss‚1.367653225733485 1loss/SRqqt1/assignment_loss‚0.815015347939201 /loss/FRt1/detection_loss‚0.5470683957134512 /loss/FBt2/detection_loss‚1.0967928092618073 % _timestamp‚1.7844626365535455e+09 _runtime‚2310.941651635 _step‚275 2loss/SRqqt2/assignment_loss‚0.7027690076010865 1loss/FBt1/assignment_loss‚0.11725849006864025 1loss/FBt2/assignment_loss‚0.18353823473876218 /loss/FBt1/detection_loss‚0.6146065971837925 &loss/total_loss‚11.394185467472692 trainer/global_step‚3349 0loss/FRt1/assignment_loss‚2.6817581250860885 1loss/SRqqt1/detection_loss‚1.1324012032497472 epoch‚8‡j¹f ® … /loss/FRt1/detection_loss‚0.5470683957134512 /loss/FBt2/detection_loss‚1.0967928092618073 % _timestamp‚1.7844626365535455e+09 _runtime‚2310.941651635 _step‚275 2loss/SRqqt2/assignment_loss‚0.7027690076010865 1loss/FBt1/assignment_loss‚0.11725849006864025 1loss/FBt2/assignment_loss‚0.18353823473876218 /loss/FBt1/detection_loss‚0.6146065971837925 &loss/total_loss‚11.394185467472692 trainer/global_step‚3349 0loss/FRt1/assignment_loss‚2.6817581250860885 1loss/SRqqt1/detection_loss‚1.1324012032497472 epoch‚8 /loss/FRt2/detection_loss‚1.4190087409394572 1loss/SRqqt2/detection_loss‚0.7163152899571726 /loss/FRt2/assignment_loss‚1.367653225733485 1loss/SRqqt1/assignment_loss‚0.815015347939201“àß|Ÿ¯ ‚(:” ºúòÒ¡Îü¤ gpu.0.gpu‚91.0 gpu.0.memory‚20, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚47.0 gpu.0.powerWatts‚114.369' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚38.123000000000005 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0g7¢t° ‚(:é ºúòÒÝŠ¥ disk.rbd0.out‚0' disk.nvme0c0n1.out‚28573.4501953125( disk.nvme1c1n1.out‚15232.00341796875 network.sent‚ 4.938235e+06" disk.dm-1.out‚1582.61083984375" disk.nvme0c0n1.in‚ 1852.4453125& disk.nvme1n1.out‚15232.00341796875$ memory_percent‚99.92382526397705' proc.memory.availableMB‚ 12.48046875 cpu‚25.846984516081037 network.recv‚634076 disk.dm-0.in‚ 1982.2109375 disk.rbd0.in‚0 disk.nvme1n1.in‚ 129.765625 disk.md0.out‚ 58.0703125 disk.dm-1.in‚0 disk.nvme1c1n1.in‚ 129.765625 disk.md0.in‚0# proc.memory.rssMB‚ 3017.04296875) proc.memory.percent‚18.41456890106201 proc.cpu.threads‚17 disk.nvme0n1.in‚ 1852.4453125% disk.nvme0n1.out‚28573.4501953125) disk./.usagePercent‚76.31377527795694$ disk./.usageGB‚1171.607234954834" disk.dm-0.out‚42097.9169921875¼þPSr± ‚(:ç ÉúòÒÐÄߤ$ disk./.usageGB‚1171.607292175293& disk.nvme0n1.out‚28574.58056640625' disk.nvme1c1n1.out‚15234.1103515625 disk.nvme1n1.in‚ 129.765625 disk.md0.out‚ 58.3046875 disk.dm-1.in‚0& proc.memory.availableMB‚ 12.7265625 proc.cpu.threads‚17 network.recv‚638095) disk./.usagePercent‚76.31377900506712 disk.nvme1c1n1.in‚ 129.765625 disk.dm-0.in‚ 1982.2109375" disk.nvme0c0n1.in‚ 1852.4453125 cpu‚25.845524442395007 network.sent‚ 4.98775e+06 disk.nvme0n1.in‚ 1852.4453125 disk.md0.in‚0" disk.dm-0.out‚42098.9013671875 disk.rbd0.in‚0 disk.rbd0.out‚0( disk.nvme0c0n1.out‚28574.58056640625% disk.nvme1n1.out‚15234.1103515625" disk.dm-1.out‚1584.35302734375$ memory_percent‚99.92232322692871# proc.memory.rssMB‚ 3017.04296875) proc.memory.percent‚18.41456890106201w-‘² ‚(:† ÉúòÒÛ·©¥ gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚47.0 gpu.0.powerWatts‚205.422' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚68.474 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0O—µ©2³ - _wandbruntime‚2342  _runtime‚2342@‰îåÇ´ Á _step‚276 & lr-AdamW/pg1‚0.0001313526482321652 & lr-AdamW/pg2‚0.0001313526482321652 trainer/global_step‚3399 % _timestamp‚1.7844626678650293e+09 _runtime‚2342.259325215/"Õ,̵ Æ & lr-AdamW/pg1‚0.0001313526482321652 & lr-AdamW/pg2‚0.0001313526482321652 trainer/global_step‚3399 % _timestamp‚1.7844626678650293e+09 _runtime‚2342.259325215 _step‚276”Óv2¶ - _wandbruntime‚2342  _runtime‚2342Ûº|%· ÚO WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚26916971 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚188 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚260393943243248 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 _runtime‚2342.259388985 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚266299825593856 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2799892594688 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 129443840 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚266299931293184 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 trainer/global_step‚3399 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚61088094 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚200223940905472 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2799981254656 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚266299931293184 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚200223717351936 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚197340549040640 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2883262420992 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚263186147573664 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2792291116842 _step‚277 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚34171123 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚263499950038528 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚61098239 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2883391864832 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚263499950038528 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚34171123 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚25754181 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚18057737 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚263499932999168 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚26916973 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 223553536 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚263186251399450 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚184 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚18057741 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2792204330416 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2799981254656 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2799892594688 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚7696440 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚26916971 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚25753993 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚266299825593856 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚266299931293184 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚200223717351936 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚197340549040640 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2883262420992 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚263186147573664 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2792291116842 _step‚277 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚34171123 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚263499950038528 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚61098239 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2883391864832 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚263499950038528 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚34171123 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚25754181 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚18057737 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚263499932999168 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚26916973 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 223553536 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚263186251399450 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚184 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚18057741 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2792204330416 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2799981254656 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2799892594688 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚7696440 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚26916971 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚25753993 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚266299825593856 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚266299825593856 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2799892594688 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 129443840 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚266299931293184 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 trainer/global_step‚3399 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚61088094 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚200223940905472 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0•6¢“Í2¹ - _wandbruntime‚2342  _runtime‚2342Eœºê&º äM E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2800766977536 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚61115702 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚26924600 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚178 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚197396686768640 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2800678310400 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚266375760975872 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚7698588 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 _runtime‚2342.664848194 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚61105543 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚263575099704832 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 121048064 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚263575082665472 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚182 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 _step‚278 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚34191100 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚25761385 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚26924600 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 % _timestamp‚1.7844626682763095e+09 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚18062793 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚34191100 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚266375866682368 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚263575082665472 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚197396592658944 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚61105543 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚61115702 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2884204352000 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2884083303936 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚263261221593656 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2792987825016 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚263575099704832 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2800678310400 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚25761203 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚200280891120640 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 _step‚278 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚34191100 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚25761385 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚26924600 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 % _timestamp‚1.7844626682763095e+09 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚18062793 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚34191100 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚266375866682368 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚263575082665472 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚197396592658944 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚61105543 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚61115702 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2884204352000 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2884083303936 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚263261221593656 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2792987825016 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚263575099704832 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2800678310400 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚25761203 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚200280891120640 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚61115702 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚26924600 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚178 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚197396686768640 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2800678310400 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚266375760975872 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚7698588 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 _runtime‚2342.664848194 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚61105543 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚263575099704832 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 121048064 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682–ª;82¼ - _wandbruntime‚2342  _runtime‚2342¯(;K½ ý 0loss/FRt2/assignment_loss‚1.1692039747541447 2loss/SRqqt2/assignment_loss‚0.6556375381898282 /loss/FRt2/detection_loss‚1.3476900862067709 epoch‚8 $ _timestamp‚1.784462668277323e+09 _runtime‚2342.665370525 &loss/total_loss‚11.556905624870506 trainer/global_step‚3399 /loss/FRt1/assignment_loss‚2.722891919059648 2loss/SRqqt1/assignment_loss‚0.7281677339745816 1loss/FBt1/assignment_loss‚0.12131989428149972 _step‚279 0loss/FBt2/assignment_loss‚0.1418904811255977 /loss/FRt1/detection_loss‚0.5429235778881484 1loss/SRqqt1/detection_loss‚1.2101640817839219 0loss/SRqqt2/detection_loss‚0.702742850338262 .loss/FBt1/detection_loss‚0.709013705439163 /loss/FBt2/detection_loss‚1.5052597818289404I|e¾ ‚ 2loss/SRqqt2/assignment_loss‚0.6556375381898282 /loss/FRt2/detection_loss‚1.3476900862067709 epoch‚8 $ _timestamp‚1.784462668277323e+09 _runtime‚2342.665370525 &loss/total_loss‚11.556905624870506 trainer/global_step‚3399 /loss/FRt1/assignment_loss‚2.722891919059648 2loss/SRqqt1/assignment_loss‚0.7281677339745816 1loss/FBt1/assignment_loss‚0.12131989428149972 _step‚279 0loss/FBt2/assignment_loss‚0.1418904811255977 /loss/FRt1/detection_loss‚0.5429235778881484 1loss/SRqqt1/detection_loss‚1.2101640817839219 0loss/SRqqt2/detection_loss‚0.702742850338262 .loss/FBt1/detection_loss‚0.709013705439163 /loss/FBt2/detection_loss‚1.5052597818289404 0loss/FRt2/assignment_loss‚1.1692039747541447—nEg0¿ ‚(:ç ØúòÒ÷•Ô¤ disk.nvme0n1.in‚ 1852.4453125 disk.dm-0.in‚ 1982.2109375" disk.dm-0.out‚42113.5322265625 disk.dm-1.out‚1584.544921875# proc.memory.rssMB‚ 3017.04296875 proc.cpu.threads‚17 network.sent‚ 4.993678e+06) disk./.usagePercent‚76.31379565282592% disk "{B./.usageGB‚1171.6075477600098" disk.nvme0c0n1.in‚ 1852.4453125& disk.nvme0n1.out‚28586.42138671875 disk.md0.in‚0 disk.dm-1.in‚0) proc.memory.percent‚18.41456890106201 network.recv‚641507( disk.nvme0c0n1.out‚28586.42138671875 disk.nvme1c1n1.in‚ 129.765625( disk.nvme1c1n1.out‚15237.98095703125 disk.nvme1n1.in‚ 129.765625 disk.rbd0.in‚0 disk.rbd0.out‚0& proc.memory.availableMB‚ 12.4296875& disk.nvme1n1.out‚15237.98095703125 disk.md0.out‚58.71875$ memory_percent‚99.92413520812988 cpu‚25.843971435251426=V IŸÀ ‚(:” ØúòÒõ¢¥ gpu.0.gpu‚91.0 gpu.0.memory‚19, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚47.0 gpu.0.powerWatts‚113.904' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚37.967999999999996 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚07!ÝgÁ ‚(:Ü çúòÒè«Ò¤" disk.nvme0c0n1.in‚ 1852.4453125' disk.nvme0c0n1.out‚28607.2529296875 proc.cpu.threads‚17" disk.dm-0.out‚42134.4697265625 disk.rbd0.in‚0 disk.rbd0.out‚0 network.sent‚ 5.05503e+06 disk.nvme1n1.in‚ 129.765625" disk.nvme1n1.out‚ 15238.5546875 disk.md0.out‚ 58.859375 disk.nvme0n1.in‚ 1852.4453125 disk.dm-0.in‚ 1982.2109375$ disk.nvme1c1n1.out‚ 15238.5546875' proc.memory.availableMB‚ 12.37890625) proc.memory.percent‚18.41456890106201 cpu‚25.84233141412531 network.recv‚645200( disk./.usagePercent‚76.2469188725593% disk./.usageGB‚1170.5808219909668 disk.md0.in‚0% disk.nvme0n1.out‚28607.2529296875 disk.nvme1c1n1.in‚ 129.765625$ memory_percent‚99.92444515228271# proc.memory.rssMB‚ 3017.04296875 disk.dm-1.in‚0 disk.dm-1.out‚1584.693359375êæÕõ“ ‚(:ˆ çúòÒæÕ¤ gpu.0.gpu‚42.0 gpu.0.memory‚10, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚47.0 gpu.0.powerWatts‚113.604' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚37.868 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0~gé2à - _wandbruntime‚2374  _runtime‚2374ðuÀ§ÈÄ Â ' lr-AdamW/pg1‚0.00013109266829257205 ' lr-AdamW/pg2‚0.00013109266829257205 trainer/global_step‚3449 % _timestamp‚1.7844627002114432e+09 _runtime‚ 2374.59986548 _step‚280>·ºÍÍÅ Ç ' lr-AdamW/pg1‚0.00013109266829257205 ' lr-AdamW/pg2‚0.00013109266829257205 trainer/global_step‚3449 % _timestamp‚1.7844627002114432e+09 _runtime‚ 2374.59986548 _step‚280˜âþ°62Æ - _wandbruntime‚2374  _runtime‚2374‹™[}Þ'Ç ØO [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2839331194880 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚18310537 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 231942144 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2839242534912 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚270096658849280 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚195 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚27298423 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚267257416314368 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚18310541 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚270096764548608 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚203071335094272 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2831442861224 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2839331194880 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚27298421 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 _step‚281 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚267257433353728 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚34663975 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚34674118 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚200147435440640 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚267257416314368 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚26114118 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2923899653632 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚266940016199858 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚270096658849280 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚34674118 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚199 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 150006272 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚267257433353728 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚26114317 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚203071103152128 trainer/global_step‚3449 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚270096764548608 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚200147341330944 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2831529647650 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚27298421 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚34663975 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2923761821184 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚264108486552208 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚264108469512848 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚27298423 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚7803581 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360Š¨2ã'È ÝO [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2839331194880 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚18310537 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 231942144 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2839242534912 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚270096658849280 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚195 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚27298423 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚267257416314368 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚18310541 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚270096764548608 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚203071335094272 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2831442861224 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2839331194880 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚27298421 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 _step‚281 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚267257433353728 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚34663975 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚34674118 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚200147435440640 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚267257416314368 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚26114118 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2923899653632 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚266940016199858 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚270096658849280 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚34674118 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚199 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 150006272 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚267257433353728 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚26114317 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚203071103152128 trainer/global_step‚3449 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚270096764548608 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚200147341330944 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2831529647650 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚27298421 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚34663975 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 2923761821184 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚264108486552208 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚264108469512848 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚27298423 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚7803581 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360™×Y]Ÿ2É - _wandbruntime‚2375  _runtime‚2375JòÁ„å$Ê äM G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚61990004 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚34683952 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚7805849 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2832226355824 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2840116917760 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 229837824 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚270172699937792 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚267332583020032 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚270172699937792 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚185 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2924574831616 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 _step‚282 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚267332583020032 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚267332565980672 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚27306052 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚34683952 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚61990004 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚34673795 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚264182777077600 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2840028250624 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚61979845 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚7805668 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚34673795 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚26121261 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚18315593 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2832313142362 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚267332565980672 RCDeviceStatsMonitor.on_train_batch_end/inactive$`ë`_split_bytes.all.peak‚ 2372707840 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 135728128 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚267015090219962 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200ÁbûÓï&Ë éM ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚34673795 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚264182777077600 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2840028250624 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚61979845 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚7805668 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚34673795 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚26121261 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚18315593 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2832313142362 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚267332565980672 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 135728128 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚267015090219962 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚61990004 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚34683952 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚7805849 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2832226355824 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2840116917760 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 229837824 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚270172699937792 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚267332583020032 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚270172699937792 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚185 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 2924574831616 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 _step‚282 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚267332583020032 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚267332565980672 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚27306052 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚34683952 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚61990004 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350šKÀö@2Ì - _wandbruntime‚2375  _runtime‚2375¡ŸÈÍ € 2loss/SRqqt1/assignment_loss‚0.7135218729973606 2loss/SRqqt2/assignment_loss‚0.6306924641320864 1loss/FBt2/assignment_loss‚0.05973396843494681 .loss/FRt2/detection_loss‚1.287322202219502 -loss/FBt2/detection_loss‚1.77378391793796 trainer/global_step‚3449 % _timestamp‚1.7844627006231344e+09 0loss/FRt1/assignment_loss‚2.6201977849006655 /loss/FRt1/detection_loss‚0.5491637331428505 1loss/SRqqt2/detection_loss‚0.6978636753450154 /loss/FBt1/detection_loss‚0.6166650036257079 _runtime‚2375.011206437 epoch‚8 _step‚283 0loss/FRt2/assignment_loss‚1.2442257901206744 2loss/FBt1/assignment_loss‚0.061391062224174675 1loss/SRqqt1/detection_loss‚1.2845983841654025 &loss/total_loss‚11.539159859246347¶Ñ Î … % _timestamp‚1.7844627006231344e+09 0loss/FRt1/assignment_loss‚2.6201977849006655 /loss/FRt1/detection_loss‚0.5491637331428505 1loss/SRqqt2/detection_loss‚0.6978636753450154 /loss/FBt1/detection_loss‚0.6166650036257079 _runtime‚2375.011206437 epoch‚8 _step‚283 0loss/FRt2/assignment_loss‚1.2442257901206744 2loss/FBt1/assignment_loss‚0.061391062224174675 1loss/SRqqt1/detection_loss‚1.2845983841654025 &loss/total_loss‚11.539159859246347 2loss/SRqqt1/assignment_loss‚0.7135218729973606 2loss/SRqqt2/assignment_loss‚0.6306924641320864 1loss/FBt2/assignment_loss‚0.05973396843494681 .loss/FRt2/detection_loss‚1.287322202219502 -loss/FBt2/detection_loss‚1.77378391793796 trainer/global_step‚3449›*psÏ ‚(:è öúòÒìºõ¤ disk.md0.in‚0 disk.dm-0.in‚ 1982.2109375 disk.dm-1.in‚0 disk.rbd0.in‚0$ memory_percent‚99.92306232452393) proc.memory.percent‚18.41456890106201 cpu‚25.841018186803144 network.sent‚ 5.072698e+06) disk./.usagePercent‚76.28209185976874& disk.nvme0n1.out‚28869.72216796875 disk.nvme1n1.in‚ 129.765625' proc.memory.availableMB‚ 12.60546875 disk.nvme0n1.in‚ 1852.4453125 disk.nvme1c1n1.in‚ 129.765625 disk.rbd0.out‚0# proc.memory.rssMB‚ 3017.04296875 proc.cpu.threads‚17$ disk.nvme1n1.out‚15294.123046875 disk.md0.out‚ 59.3984375% disk./.usageGB‚1171.1208152770996" disk.nvme0c0n1.in‚ 1852.4453125( disk.nvme0c0n1.out‚28869.72216796875& disk.nvme1c1n1.out‚15294.123046875# disk.dm-0.out‚42450.98681640625! disk.dm-1.out‚1585.0673828125 network.recv‚648984òÑifœÐ ‚(:‘ öúòÒ­¨¥ gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚48.0 gpu.0.powerWatts‚113.755' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚37.91833333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0²¤PïÑ ‚(:’ …ûòÒù´Ú¤ gpu.0.gpu‚11.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚49.0 gpu.0.powerWatts‚225.989' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚75.32966666666667 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Â-7mÒ ‚(:â …ûòÒÐï²¥ disk.rbd0.in‚0 disk.nvme0n1.in‚ 1852.4453125% disk.nvme0n1.out‚29001.1826171875' disk.nvme1c1n1.out‚15353.5869140625 disk.nvme1c1n1.in‚ 129.765625& proc.memory.availableMB‚ 12.6015625# proc.memory.rssMB‚ 3017.04296875 cpu‚25.83940579626326) disk./.usagePercent‚76.27781959761045" disk.dm-0.out‚42641.2451171875 disk.dm-1.in‚0% disk.nvme1n1.out‚15353.5869140625) proc.memory.percent‚18.41456890106201 proc.cpu.threads‚17 network.sent‚ 5.124126e+06 network.recv‚653173 disk.dm-0.in‚ 1982.2109375 disk.rbd0.out‚0" disk.nvme0c0n1.in‚ 1852.4453125' disk.nvme0c0n1.out‚29001.1826171875 disk.nvme1n1.in‚ 129.765625$ memory_percent‚99.92308616638184% disk./.usageGB‚1171.0552253723145 disk.md0.in‚0 disk.md0.out‚59.6875! disk.dm-1.out‚1585.1025390625ù”T2Ó - _wandbruntime‚2406  _runtime‚2406"è&ÆÔ À _runtime‚2406.863865359 _step‚284 & lr-AdamW/pg1‚0.0001308287401615606 & lr-AdamW/pg2‚0.0001308287401615606 trainer/global_step‚3499 $ _timestamp‚1.784462732473929e+09¹[ûcËÕ Å & lr-AdamW/pg2‚0.0001308287401615606 trainer/global_step‚3499 $ _timestamp‚1.784462732473929e+09 _runtime‚2406.863865359 _step‚284 & lr-AdamW/pg1‚0.0001308287401615606œe ÿÌ2Ö - _wandbruntime‚2406  _runtime‚2406‘šHÃà'× ÚO L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚27679871 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚270693677174480 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚35156827 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚271014899629568 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 225650688 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2878681135104 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚273893492104704 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚62846843 % _timestamp‚1.7844627324746437e+09 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚205918669776384 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚273893492104704 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2870681392032 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 131540992 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2878592475136 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 _runtime‚2406.863959159 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚18563341 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2964347935744 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚273893597804032 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚26473152 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚7910004 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚189 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚271014899629568 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚27679873 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚193 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚273893597804032 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2878592475136 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚270693781000266 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 2964347935744 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚273893597804032 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚26473152 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚7910004 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚189 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚271014899629568 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚27679873 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚193 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚273893597804032 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2878592475136 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚270693781000266 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚273893492104704 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2870681392032 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 131540992 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2878592475136 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 _runtime‚2406.863959159 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚18563341Pªe2Ù - _wandbruntime‚2407  _runtime‚2407OÆø$ê&Ú äM ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚205975610415104 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚62854147 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚27687500 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚35176804 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 _step‚286 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2879378190848 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2879378190848 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚62864306 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2871551673170 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚271090049295872 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚35176804 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚271090066335232 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚35166647 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚273969427486720 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚27687500 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚181 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 219352064 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚205975391063040 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2965150846464 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚26480317 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚7912101 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚62864306 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚62854147 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚26480498 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚270768751194472J©åMï&Û éM C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚270768751194472 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚205975610415104 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚62854147 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚27687500 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚35176804 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 _step‚286 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2879378190848 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2879378190848 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚62864306 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2871551673170 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚271090049295872 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚35176804 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚271090066335232 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚35166647 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚273969427486720 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚27687500 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚181 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 219352064 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚205975391063040 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 2965150846464 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚26480317 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚7912101 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚62864306 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚62854147 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚26480498 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0žÌ3¹º2Ü - _wandbruntime‚2407  _runtime‚24077*1Ý € _step‚287 2loss/SRqqt2/assignment_loss‚0.7687176282228292 &loss/total_loss‚12.233875149474375 2loss/SRqqt1/assignment_loss‚0.6688037425730982 1loss/FBt1/assignment_loss‚0.11898512940746811 /loss/FBt1/detection_loss‚0.9934614340726986 epoch‚8 _runtime‚2407.270680172 0loss/FRt2/assignment_loss‚1.1934141843863155 1loss/FBt2/assignment_loss‚0.22161354650893372 /loss/FRt1/detection_loss‚0.5348909825398529 1loss/SRqqt2/detection_loss‚0.7118702439390439 /loss/FBt2/detection_loss‚1.7545893451571464 /loss/FRt1/assignment_loss‚2.809814089420302 /loss/FRt2/detection_loss‚1.4249303360339727 0loss/SRqqt1/detection_loss‚1.032784487212714 trainer/global_step‚3499 % _timestamp‚1.7844627328826258e+09\—HD Þ … 1loss/FBt2/assignment_loss‚0.22161354650893372 /loss/FRt1/detection_loss‚0.5348909825398529 1loss/SRqqt2/detection_loss‚0.7118702439390439 /loss/FBt2/detection_loss‚1.7545893451571464 /loss/FRt1/assignment_loss‚2.809814089420302 /loss/FRt2/detection_loss‚1.4249303360339727 0loss/SRqqt1/detection_loss‚1.032784487212714 trainer/global_step‚3499 % _timestamp‚1.7844627328826258e+09 _step‚287 2loss/SRqqt2/assignment_loss‚0.7687176282228292 &loss/total_loss‚12.233875149474375 2loss/SRqqt1/assignment_loss‚0.6688037425730982 1loss/FBt1/assignment_loss‚0.11898512940746811 /loss/FBt1/detection_loss‚0.9934614340726986 epoch‚8 _runtime‚2407.270680172 0loss/FRt2/assignment_loss‚1.1934141843863155Ÿ‡šŒöjß ‚(:ß ”ûòÒ·ƒû¤ network.recv‚656093% disk./.usageGB‚1171.0551872253418' disk.nvme1c1n1.out‚15355.7939453125 disk.nvme1n1.in‚ 129.765625% disk.nvme1n1.out‚15355.7939453125 disk.md0.in‚0# proc.memory.rssMB‚ 3017.04296875) proc.memory.percent‚18.41456890106201 network.sent‚ 5.139762e+06) disk./.usagePercent‚76.27781711287034 disk.dm-0.in‚ 1982.2109375! disk.dm-0.out‚42650.912109375 disk.rbd0.in‚0" disk.nvme0c0n1.in‚ 1852.4453125 disk.nvme0n1.in‚ 1852.4453125 disk.nvme1c1n1.in‚ 129.765625 disk.dm-1.in‚0! disk.dm-1.out‚1585.2998046875 disk.rbd0.out‚0# disk.nvme0c0n1.out‚ 29009.859375! disk.nvme0n1.out‚ 29009.859375 disk.md0.out‚ 60.17578125$ memory_percent‚99.91624355316162' proc.memory.availableMB‚ 13.72265625 cpu‚25.837936775077974 proc.cpu.threads‚17mÑ°žà ‚(:“ ”ûòÒ’Ž¨¥ gpu.0.gpu‚91.0 gpu.0.memory‚20, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚48.0 gpu.0.powerWatts‚114.427' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚38.14233333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Up›ßiá ‚(:Þ £ûòÒñûȤ% disk./.usageGB‚1172.0004043579102 disk.nvme1c1n1.in‚ 129.765625' disk.nvme1c1n1.out‚15381.3408203125 disk.nvme1n1.in‚ 129.765625 cpu‚25.836280166736532 network.recv‚660247" disk.nvme0c0n1.in‚ 1852.4453125% disk.nvme0c0n1.out‚29598.80078125 disk.md0.in‚0! disk.dm-0.out‚43264.244140625 disk.rbd0.out‚0$ memory_percent‚99.91779327392578$ proc.memory.availableMB‚13.46875 network.sent‚ 5.190556e+06 disk.md0.out‚ 60.453125 disk.dm-0.in‚ 1982.2109375 disk.dm-1.in‚0! disk.dm-1.out‚1585.8310546875# proc.memory.rssMB‚ 3017.04296875 proc.cpu.threads‚17) disk./.usagePercent‚76.33938474892754 disk.nvme0n1.in‚ 1852.4453125# disk.nvme0n1.out‚29598.80078125% disk.nvme1n1.out‚15381.3408203125 disk.rbd0.in‚0) proc.memory.percent‚18.41456890106201_"D‚žâ ‚(:“ £ûòÒêƹ¤ gpu.0.gpu‚76.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚49.0 gpu.0.powerWatts‚223.313' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚74.43766666666666 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0A–Ï2ã - _wandbruntime‚2438  _runtime‚2438v©æQÉä à _step‚288 ' lr-AdamW/pg1‚0.00013056088079074158 ' lr-AdamW/pg2‚0.00013056088079074158 trainer/global_step‚3549 % _timestamp‚1.7844627643585334e+09 _runtime‚2438.746912907œ¦éÎå È ' lr-AdamW/pg1‚0.00013056088079074158 ' lr-AdamW/pg2‚0.00013056088079074158 trainer/global_step‚3549 % _timestamp‚1.7844627643585334e+09 _runtime‚2438.746912907 _step‚288 ãØ=2æ - _wandbruntime‚2438  _runtime‚2438t~Â:Åç ÚO G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚274772399984128 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚277690431059456 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3004617974272 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚28061323 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚277690325360128 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚18816141 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚8015756 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 223553536 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚26831897 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚35649679 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 150006272 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 _step‚289 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚205761208240640 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚274447545800674 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚28061321 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚208765955658752 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚271537539091408 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚277690325360128 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚277690431059456 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2917942415360 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚35659822 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚274772399984128 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚277690431059456 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3004617974272 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚28061323 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚277690325360128 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚18816141 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚8015756 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 223553536 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚26831897 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚35649679 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 150006272 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 _step‚289 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚205761208240640 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚274447545800674 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚28061321 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚208765955658752 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚271537539091408 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚277690325360128 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚277690431059456 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 2917942415360 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚35659822 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3005563943936 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚274847549650432 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚274847532611072 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚18821193 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚271611812577440 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚63728449 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 trainer/global_step‚3549 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚63738608 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚205817251858944 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚271611829616800 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚8017859 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚208822909912576 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚28068952 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚274522619820778 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚277766366448640 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2918816798208 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2918728131072 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚35669656 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚186 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 225643520 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚26839056 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚208822684269056 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚28068950 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2910790203978 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2918728131072 _step‚290 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚28068950 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚35669656 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚277766260742144 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 150006272 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚18821197 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚205817345968640 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚26838866 % _timestamp‚1.7844627647693925e+09 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚274847549650432 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚63738608 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚274522515994880Ÿ´E<Áë éM RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚35669656 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚186 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 225643520 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚26839056 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚208822684269056 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚28068950 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2910790203978 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2918728131072 _step‚290 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚28068950 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚35669656 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚277766260742144 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 150006272 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚18821197 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚205817345968640 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚26838866 % _timestamp‚1.7844627647693925e+09 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚274847549650432 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚63738608 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚274522515994880 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3005563943936 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚274847549650432 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚274847532611072 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚18821193 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚271611812577440 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚63728449 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 trainer/global_step‚3549 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚63738608 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚205817251858944 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚271611829616800 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚8017859 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚208822909912576 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚28068952 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚274522619820778 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚277766366448640 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2918816798208 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 2918728131072 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2¢Jæ{f2ì - _wandbruntime‚2439  _runtime‚2439r'-‹í  /loss/FRt1/assignment_loss‚2.759559427079555 2loss/SRqqt1/assignment_loss‚0.6947362296580435 /loss/FBt2/detection_loss‚1.3264528661966324 epoch‚8 % _timestamp‚1.7844627647708266e+09 0loss/FRt2/assignment_loss‚1.2264227517573378 1loss/FBt2/assignment_loss‚0.29421684459026876 1loss/SRqqt1/detection_loss‚1.0870486084500524 1loss/SRqqt2/detection_loss‚0.7654221882583305 trainer/global_step‚3549 2loss/SRqqt2/assignment_loss‚0.6202694257564911 1loss/FBt1/assignment_loss‚0.08993197201731364 /loss/FRt1/detection_loss‚0.5473580652597698 /loss/FRt2/detection_loss‚1.5032054585590522 /loss/FBt1/detection_loss‚0.8024610268318777 &loss/total_loss‚11.717084864414726 _runtime‚2439.159015264 _step‚291²£† î † 2loss/SRqqt2/assignment_loss‚0.6202694257564911 1loss/FBt1/assignment_loss‚0.08993197201731364 /loss/FRt1/detection_loss‚0.5473580652597698 /loss/FRt2/detection_loss‚1.5032054585590522 /loss/FBt1/detection_loss‚0.8024610268318777 &loss/total_loss‚11.717084864414726 _runtime‚2439.159015264 _step‚291 /loss/FRt1/assignment_loss‚2.759559427079555 2loss/SRqqt1/assignment_loss‚0.6947362296580435 /loss/FBt2/detection_loss‚1.3264528661966324 epoch‚8 % _timestamp‚1.7844627647708266e+09 0loss/FRt2/assignment_loss‚1.2264227517573378 1loss/FBt2/assignment_loss‚0.29421684459026876 1loss/SRqqt1/detection_loss‚1.0870486084500524 1loss/SRqqt2/detection_loss‚0.7654221882583305 trainer/global_step‚3549£¶ßl'qï ‚(:æ ²ûòÒίí¤ network.recv‚663849# disk./.usageGB‚1172.00341796875' disk.nvme0c0n1.out‚29959.8037109375 disk.nvme0n1.in‚ 1852.4453125 disk.nvme1c1n1.in‚ 129.765625 disk.dm-1.in‚0 disk.rbd0.out‚0" disk.nvme0c0n1.in‚ 1852.4453125% disk.nvme0n1.out‚29959.8037109375( disk.nvme1c1n1.out‚15382.88232421875 disk.md0.in‚0! disk.dm-0.out‚43625.951171875$ memory_percent‚99.91669654846191& proc.memory.availableMB‚ 13.6484375# proc.memory.rssMB‚ 3017.04296875 network.sent‚ 5.193478e+06) disk./.usagePercent‚76.33958104339685 disk.nvme1n1.in‚ 129.765625 disk.dm-0.in‚ 1982.2109375) proc.memory.percent‚18.41456890106201 proc.cpu.threads‚17& disk.nvme1n1.out‚15382.88232421875" disk.dm-1.out‚1586.06103515625 disk.rbd0.in‚0 cpu‚25.834730548486625 disk.md0.out‚ 60.7265625³õAIžð ‚(:“ ²ûòÒ£‚‘¥ gpu.0.gpu‚71.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚49.0 gpu.0.powerWatts‚219.542' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚73.18066666666667 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ÚȽ©oñ ‚(:ä ÁûòÒ¡èǤ cpu‚25.83361671657423% disk./.usageGB‚1172.0038146972656 disk.nvme1n1.in‚ 129.765625! disk.dm-0.out‚43626.564453125" disk.dm-1.out‚1589.28369140625( disk.nvme0c0n1.out‚29960.62841796875& disk.nvme0n1.out‚29960.62841796875' proc.memory.availableMB‚ 13.64453125) disk./.usagePercent‚76.33960688469406% disk.nvme1c1n1.out‚15386.54296875# disk.nvme1n1.out‚15386.54296875 disk.md0.out‚ 61.0234375 disk.dm-1.in‚0 disk.rbd0.out‚0 disk.nvme0n1.in‚ 1852.4453125 network.sent‚ 5.257771e+06 disk.dm-0.in‚ 1982.2109375" disk.nvme0c0n1.in‚ 1852.4453125$ memory_percent‚99.91672039031982) proc.memory.percent‚18.41456890106201 proc.cpu.threads‚17 network.recv‚667909 disk.nvme1c1n1.in‚ 129.765625 disk.md0.in‚0 disk.rbd0.in‚0# proc.memory.rssMB‚ 3017.04296875BIêü“ò ‚(:ˆ ÁûòÒµ»¥ gpu.0.gpu‚91.0 gpu.0.memory‚21, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚48.0 gpu.0.powerWatts‚208.344' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚69.448 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0í Ü 2ó - _wandbruntime‚2471  _runtime‚2471zåLóÇô Á & lr-AdamW/pg1‚0.0001302891073842218 & lr-AdamW/pg2‚0.0001302891073842218 trainer/global_step‚3599 % _timestamp‚1.7844627968131142e+09 _runtime‚2471.201482097 _step‚292DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚36142531 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2949158453648 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚278529883299328 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚64595447 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚28442771 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 % _timestamp‚1.7844627968139644e+09 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚281487158615552 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚36142531 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 225650688 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚36152674 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 131540992 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 _step‚293 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚275252048321648 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚278529866259968 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2957381015552 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚64585302 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2949245240074 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚281487264314880 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚28442773 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚28442771 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚19068937 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚281487158615552 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2957292355584 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3045177448960 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚208568094640640 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3045045907968 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚275252065361008 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚174 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚27190682 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 150006272 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚8121745 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚28442773 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚281487264314880 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2957381015552 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚27190856 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚278201206775296 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚278529866259968 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚278529883299328 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 _runtime‚2471.205820517 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚211613272089600 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚208568000530944 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552?ú;Iå'ø ßO ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚211613272089600 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚208568000530944 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚36142531 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 2949158453648 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚278529883299328 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚64595447 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚28442771 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 % _timestamp‚1.7844627968139644e+09 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚281487158615552 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚36142531 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 225650688 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚36152674 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 131540992 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 _step‚293 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚275252048321648 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚278529866259968 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 2957381015552 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚64585302 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 2949245240074 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚281487264314880 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚28442773 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚28442771 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚19068937 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚281487158615552 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 2957292355584 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3045177448960 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚208568094640640 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3045045907968 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚275252065361008 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚174 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚27190682 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 150006272 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚8121745 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚28442773 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚281487264314880 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 2957381015552 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚27190856 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚278201206775296 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚278529866259968 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚278529883299328 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 _runtime‚2471.205820517 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303¥”<úÄ2ù - _wandbruntime‚2471  _runtime‚2471ÆuL\ú äM ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2958166738432 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 _step‚294 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2958166738432 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚19073997 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 _runtime‚2471.605486776 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚278605032965632 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚36162508 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 231934976 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚19073993 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2949941948248 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚28450402 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚64612910 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚8124011 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚278276280795288 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2950028734786 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚211669988633088 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 150006272 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚27198008 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚36162508 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 137825280 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3045850374144 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚28450400 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚281563093997568 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚36152351 E=Device©ÌStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚208624138258944 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚278605032965632 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚36152351 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2958078071296 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚171 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚208624232368640 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚278276384621186 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚275326355886400 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚64602751 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚27197837 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 % _timestamp‚1.7844627972169852e+09 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚275326338847040 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3045988199424 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8123844 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 trainer/global_step‚3599 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚211670220568064 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136€wº&ï&û éM UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚64612910 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚28450402 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚28450400 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚278605015926272 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚27197837 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 % _timestamp‚1.7844627972169852e+09 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚275326338847040 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3045988199424 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8123844 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 trainer/global_step‚3599 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚211670220568064 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 2958166738432 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 _step‚294 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 2958166738432 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚19073997 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 _runtime‚2471.605486776 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚278605032965632 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚36162508 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 231934976 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚19073993 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 2949941948248 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚28450402 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚64612910 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚8124011 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚278276280795288 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 2950028734786 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚211669988633088 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 150006272 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚27198008 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚36162508 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 137825280 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3045850374144 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚28450400 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚281563093997568 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚36152351 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚208624138258944 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚278605032965632 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚36152351 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 2958078071296 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚171 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚208624232368640 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚278276384621186 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚275326355886400 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚64602751 E&Purity/1FRt*SRqqt1FBt/event_proportion‚0.01643880270421505 ,&Purity/2FRt2SRqqt1FBt/event_proportion‚0 9"Purity/*FRt*SRqqt1FBt/event_purity‚0.6999806761741638 8&Purity/*FRt2SRqqt0FBt/event_proportion‚ 0.05322265625 @&Purity/1FRt0SRqqt2FBt/event_proportion‚0.0024879092816263437 8"Purity/1FRt1SRqqt1FBt/SRqqt_purity‚0.641708254814148 :"Purity/2FRt*SRqqt*FBt/event_purity‚0.01799742318689823 @&Purity/2FRt*SRqqt2FBt/event_proportion‚0.0002092633949359879 7 Purity/*FRt0SRqqt*FBt/FBt_purity‚0.9894615411758423 9"Purity/0FRt0SRqqt1FBt/event_purity‚0.9970456957817078 :"Purity/1FRt2SRqqt1FBt/SRqqt_purity‚0.32499998807907104 8 Purity/*FRt0SRqqt*FBt/FRt_purity‚0.14942173659801483 @&Purity/2FRt1SRqqt2FBt/event_proportion‚2.325148852833081e-05 & Purity/2FRt1SRqqt2FBt/FBt_purity‚1 jet/accuracy_1_of_5‚1 6 Purity/*FRt1SRqqt*FBt/FBt_purity‚0.984939694404602 & Purity/*FRt2SRqqt1FBt/FRt_purity‚0 ("Purity/1FRt2SRqqt1FBt/event_purity‚0 9"Purity/2FRt2SRqqt0FBt/SRqqt_purity‚0.8465028405189514 7 Purity/0FRt1SRqqt2FBt/FBt_purity‚0.9230769276618958 7 Purity/*FRt*SRqqt0FBt/FRt_purity‚0.1291438639163971 7 Purity/*FRt*SRqqt2FBt/FRt_purity‚0.8512850403785706 8 Purity/*FRt1SRqqt*FBt/FRt_purity‚0.14545424282550812 ("Purity/*FRt1SRqqt2FBt/event_purity‚0 9"Purity/0FRt0SRqqt*FBt/event_purity‚0.9849008917808533 8"Purity/0FRt2SRqqt0FBt/SRqqt_purity‚0.621488630771637 @&Purity/1FRt*SRqqt2FBt/event_proportion‚0.0027204242069274187 7 Purity/1FRt1SRqqt*FBt/FBt_purity‚0.9872215986251831 7 Purity/*FRt*SRqqt1FBt/FBt_purity‚0.9953683018684387 7 Purity/1FRt1SRqqt*FBt/FRt_purity‚0.1315348744392395 trainer/global_step‚3599 9"Purity/0FRt*SRqqt0FBt/event_purity‚0.5999293923377991 =&Purity/0FRt*SRqqt1FBt/event_proportion‚0.0436662957072258 =&Purity/0FRt0SRqqt0FBt/event_proportion‚0.0462472103536129 9"Purity/0FRt1SRqqt0FBt/SRqqt_purity‚0.6228646636009216 jet/accuracy_4_of_5‚0 8&Purity/1FRt2SRqqt0FBt/event_proportion‚ 0.02294921875 <&Purity/2FRt*SRqqt*FBt/event_proportion‚0.040062315762043 ("Purity/2FRt*SRqqt2FBt/SRqqt_purity‚0 <&Purity/*FRt*SRqqt0FBt/event_proportion‚0.881045401096344 7 Purity/*FRt0SRqqt0FBt/FRt_purity‚0.1350795179605484 7 Purity/*FRt0SRqqt2FBt/FRt_purity‚0.8643235564231873 ("Purity/0FRt0SRqqt0FBt/event_purity‚1 =&Purity/1FRt*SRqqt0FBt/event_proportion‚0.6105840802192688 9"Purity/*FRt*SRqqt2FBt/event_purity‚0.8186385035514832 particle/accuracy_1_of_3‚1 8 Purity/*FRt*SRqqt*FBt/FRt_purity‚0.14313514530658722 8 Purity/*FRt1SRqqt0FBt/FRt_purity‚0.12982788681983948 ) Purity/*FRt1SRqqt2FBt/FRt_purity‚0.75 :"Purity/1FRt0SRqqt*FBt/event_purity‚0.14746864140033722 & Purity/1FRt1SRqqt1FBt/FBt_purity‚1 ("Purity/2FRt1SRqqt0FBt/event_purity‚0 particle/accuracy_2_of_4‚1 6validation_average_jet_accuracy‚0.3389193117618561 7 Purity/*FRt*SRqqt*FBt/FBt_purity‚0.9882121086120605 =&Purity/*FRt1SRqqt0FBt/event_proportion‚0.3670479953289032 @&Purity/*FRt1SRqqt2FBt/event_proportion‚0.0006975446594879031 >&Purity/2FRt0SRqqt0FBt/event_proportion‚0.01990327425301075 7 Purity/*FRt1SRqqt1FBt/FBt_purity‚0.9936917424201965 9"Purity/1FRt*SRqqt2FBt/event_purity‚0.6751889586448669 7 Purity/1FRt0SRqqt1FBt/FRt_purity‚0.5823805332183838 8 Purity/1FRt1SRqqt0FBt/FRt_purity‚0.11597421020269394 7 Purity/1FRt1SRqqt2FBt/FRt_purity‚0.7777777910232544 A&Purity/2FRt0SRqqt2FBt/event_proportion‚0.00018601190822664648 9"Purity/*FRt2SRqqt0FBt/SRqqt_purity‚0.6993999481201172 0"Purity/0FRt2SRqqt1FBt/SRqqt_purity‚ 0.3984375 @&Purity/2FRt1SRqqt1FBt/event_proportion‚0.0008370535797439516 7 Purity/*FRt0SRqqt1FBt/FBt_purity‚0.9965531229972839 7 Purity/0FRt*SRqqt2FBt/FBt_purity‚0.9626487493515015 :"Purity/2FRt0SRqqt0FBt/event_purity‚0.01834227330982685 & Purity/2FRt*SRqqt2FBt/FBt_purity‚1 particle/accuracy_1_of_5‚1 =&Purity/*FRt0SRqqt1FBt/event_proportion‚0.0418526791036129 =&Purity/0FRt0SRqqt1FBt/event_proportion‚0.0313895083963871 ("Purity/0FRt2SRqqt1FBt/event_purity‚0 9"Purity/2FRt0SRqqt2FBt/event_purity‚0.5714285969734192 9"Purity/2FRt*SRqqt*FBt/SRqqt_purity‚0.7864935994148254 9"Purity/0FRt*SRqqt2FBt/SRqqt_purity‚0.1538461595773697 ("Purity/2FRt1SRqqt2FBt/event_purity‚0 ("Purity/*FRt2SRqqt1FBt/event_purity‚0 :"Purity/1FRt1SRqqt0FBt/event_purity‚0.03233988955616951 9"Purity/1FRt1SRqqt2FBt/SRqqt_purity‚0.3333333432674408 9"Purity/*FRt1SRqqt0FBt/SRqqt_purity‚0.6714556813240051 8 Purity/*FRt2SRqqt0FBt/FRt_purity‚0.04693333059549332 A&Purity/1FRt2SRqqt1FBt/event_proportion‚0.00027901786961592734 +jet/accuracy_3_of_4‚0.07620833069086075 7 Purity/*FRt*SRqqt1FBt/FRt_purity‚0.5190966129302979 :"Purity/*FRt1SRqqt*FBt/event_purity‚0.36532261967658997 :"Purity/*FRt2SRqqt1FBt/SRqqt_purity‚0.35879629850387573 ?&Purity/0FRt0SRqqt2FBt/event_proportion‚0.007626487873494625 =&Purity/2FRt1SRqqt*FBt/event_proportion‚0.0127883180975914 particle/accuracy_1_of_1‚1 % _timestamp‚1.7844628107683563e+09 7 Purity/*FRt2SRqqt1FBt/FBt_purity‚0.9814814329147339 7 Purity/1FRt0SRqqt*FBt/FBt_purity‚0.9941900372505188 :"Purity/1FRt0SRqqt0FBt/event_purity‚0.13610389828681946 9"Purity/2FRt*SRqqt0FBt/SRqqt_purity‚0.7900545001029968 9"Purity/0FRt*SRqqt1FBt/event_purity‚0.8358891606330872 8 Purity/1FRt0SRqqt*FBt/FRt_purity‚0.14962711930274963 =&Purity/*FRt0SRqqt*FBt/event_proportion‚0.5129278302192688 =&Purity/0FRt0SRqqt*FBt/event_proportion‚0.0390159972012043 , Purity/0FRt2SRqqt1FBt/FBt_purity‚0.96875 9"Purity/1FRt1SRqqt*FBt/SRqqt_purity‚0.7303345799446106 *jet/accuracy_2_of_3‚0.5495484471321106 9"Purity/*FRt*SRqqt0FBt/SRqqt_purity‚0.6776919364929199 9"Purity/0FRt*SRqqt1FBt/SRqqt_purity‚0.5051735639572144 ("Purity/1FRt2SRqqt*FBt/event_purity‚0 =&Purity/*FRt*SRqqt*FBt/event_proportion‚0.9537528157234192 >&Purity/1FRt2SRqqt*FBt/event_proportion‚0.02322823740541935 8 Purity/1FRt2SRqqt0FBt/FRt_purity‚0.05019146949052811 9"Purity/2FRt1SRqqt0FBt/SRqqt_purity‚0.7365633845329285 *jet/accuracy_1_of_3‚0.9184180498123169 'particle/accuracy‚0.840839684009552 :"Purity/*FRt*SRqqt2FBt/SRqqt_purity‚0.24074071645736694 =&Purity/*FRt1SRqqt1FBt/event_proportion‚0.0188569575548172 9"Purity/*FRt1SRqqt1FBt/SRqqt_purity‚0.5697539448738098 =&Purity/0FRt*SRqqt0FBt/event_proportion‚0.2322126179933548 8"Purity/0FRt1SRqqt1FBt/SRqqt_purity‚0.522489070892334 =&Purity/1FRt*SRqqt*FBt/event_proportion‚0.6297432780265808 9"Purity/1FRt0SRqqt2FBt/event_purity‚0.7392479777336121 +particle/sensitivity‚0.6298382878303528 9"Purity/*FRt1SRqqt1FBt/event_purity‚0.3556191921234131 :"Purity/0FRt1SRqqt1FBt/event_purity‚0.45889630913734436 :"Purity/2FRt0SRqqt*FBt/event_purity‚0.03457769751548767 particle/accuracy_1_of_4‚1 jet/accuracy_2_of_5‚1 _step‚296 :"Purity/*FRt*SRqqt*FBt/event_purity‚0.26909762620925903 7 Purity/0FRt0SRqqt*FBt/FBt_purity‚0.9874101281166077 :"Purity/1FRt*SRqqt1FBt/event_purity‚0.39110028743743896 9"Purity/1FRt*SRqqt2FBt/SRqqt_purity‚0.3333333432674408 =&Purity/1FRt1SRqqt*FBt/event_proportion‚0.1534598171710968 8 Purity/2FRt0SRqqt*FBt/FRt_purity‚0.14758087694644928 7 Purity/2FRt1SRqqt1FBt/FRt_purity‚0.4490741193294525 =&Purity/*FRt0SRqqt0FBt/event_proportion‚0.4607747495174408 :"Purity/0FRt2SRqqt*FBt/event_purity‚0.39385467767715454 >&Purity/0FRt2SRqqt*FBt/event_proportion‚0.02457682229578495 9"Purity/2FRt2SRqqt*FBt/SRqqt_purity‚0.8465028405189514 & Purity/2FRt0SRqqt*FBt/FBt_purity‚1 =&Purity/*FRt*SRqqt1FBt/event_proportion‚0.0617094486951828 9"Purity/*FRt2SRqqt*FBt/SRqqt_purity‚0.6931633949279785 :"Purity/1FRt*SRqqt*FBt/event_purity‚0.11588989198207855 7 Purity/1FRt*SRqqt*FBt/FBt_purity‚0.9918419718742371 9"Purity/1FRt*SRqqt1FBt/SRqqt_purity‚0.6224424839019775 & Purity/1FRt2SRqqt1FBt/FBt_purity‚1 (jet/accuracy_1_of_1‚0.28925821185112 :"Purity/*FRt2SRqqt0FBt/event_purity‚0.17934942245483398 8 Purity/1FRt*SRqqt*FBt/FRt_purity‚0.14154718816280365 =&Purity/1FRt1SRqqt0FBt/event_proportion‚0.1467633992433548 *jet/accuracy_1_of_4‚0.9684416651725769 particle/accuracy_2_of_3‚1 9"Purity/*FRt1SRqqt0FBt/event_purity‚0.3666115403175354 9"Purity/0FRt*SRqqt2FBt/event_purity‚0.8772681951522827 ("Purity/2FRt1SRqqt*FBt/event_purity‚0 ("Purity/2FRt1SRqqt2FBt/SRqqt_purity‚0 <&Purity/*FRt2SRqqt*FBt/event_proportion‚0.054222472012043 ("Purity/1FRt1SRqqt2FBt/event_purity‚0 7 Purity/1FRt1SRqqt2FBt/FBt_purity‚0.7777777910232544 =&Purity/2FRt*SRqqt0FBt/event_proportion‚0.0382486991584301 9"Purity/2FRt*SRqqt1FBt/SRqqt_purity‚0.7314814329147339 9 Purity/2FRt2SRqqt0FBt/FRt_purity‚0.036971911787986755 =&Purity/*FRt1SRqqt*FBt/event_proportion‚0.3866024911403656 9"Purity/0FRt1SRqqt*FBt/event_purity‚0.6128073930740356 9"Purity/1FRt1SRqqt0FBt/SRqqt_purity‚0.7352995872497559 8 Purity/2FRt*SRqqt0FBt/FRt_purity‚0.13830778002738953 :"Purity/2FRt*SRqqt1FBt/event_purity‚0.16500000655651093 /particle/accuracy_4_of_4‚0.9759465456008911 ) Purity/2FRt*SRqqt2FBt/FRt_purity‚0.75 & Purity/2FRt1SRqqt*FBt/FBt_purity‚1 :"Purity/*FRt*SRqqt0FBt/event_purity‚0.23201200366020203 7 Purity/*FRt0SRqqt2FBt/FBt_purity‚0.9742013812065125 ,&Purity/*FRt2SRqqt2FBt/event_proportion‚0 5 Purity/0FRt*SRqqt1FBt/FBt_purity‚0.99461829662323 & Purity/2FRt*SRqqt1FBt/FBt_purity‚1 8 Purity/2FRt1SRqqt*FBt/FRt_purity‚0.22816292941570282 7 Purity/2FRt0SRqqt1FBt/FRt_purity‚0.6197916269302368 @&Purity/*FRt2SRqqt1FBt/event_proportion‚0.0009998140158131719 9"Purity/0FRt*SRqqt*FBt/SRqqt_purity‚0.6158679127693176 7 Purity/1FRt*SRqqt1FBt/FRt_purity‚0.5208590626716614 @&Purity/1FRt1SRqqt1FBt/event_proportion‚0.0064639137126505375 ;"Purity/2FRt*SRqqt0FBt/event_purity‚0.009579349309206009 7 Purity/*FRt1SRqqt2FBt/FBt_purity‚0.8518518209457397 6 Purity/0FRt1SRqqt*FBt/FBt_purity‚0.984364926815033 =&Purity/0FRt1SRqqt0FBt/event_proportion‚0.2083565890789032 @&Purity/0FRt1SRqqt2FBt/event_proportion‚0.0004417782765813172 =&Purity/2FRt0SRqqt*FBt/event_proportion‚0.0208565853536129 9"Purity/0FRt*SRqqt*FBt/event_purity‚0.6443859934806824 ("Purity/1FRt2SRqqt0FBt/event_purity‚0 & Purity/1FRt2SRqqt1FBt/FRt_purity‚0 9 Purity/2FRt2SRqqt*FBt/FRt_purity‚0.036971911787986755 7 Purity/0FRt1SRqqt1FBt/FBt_purity‚0.9900330901145935 @&Purity/0FRt2SRqqt1FBt/event_proportion‚0.0007207961170934141 9"Purity/1FRt*SRqqt0FBt/SRqqt_purity‚0.7375082969665527 /particle/accuracy_2_of_2‚0.9998962879180908 ("Purity/0FRt1SRqqt2FBt/event_purity‚0 *loss/val_total_loss‚11.003473853957328 =&Purity/*FRt0SRqqt2FBt/event_proportion‚0.0103004090487957 8 Purity/2FRt*SRqqt*FBt/FRt_purity‚0.15644004940986633 & Purity/2FRt1SRqqt1FBt/FBt_purity‚1 +jet/accuracy_2_of_2‚0.13320466876029968 6 Purity/0FRt*SRqqt*FBt/FBt_purity‚0.986178994178772 & Purity/2FRt*SRqqt*FBt/FBt_purity‚1 8 Purity/2FRt1SRqqt0FBt/FRt_purity‚0.21442243456840515 ("Purity/2FRt1SRqqt1FBt/event_purity‚0 +particle/specificity‚0.8985463976860046 ( Purity/2FRt1SRqqt2FBt/FRt_purity‚0.5 9"Purity/0FRt1SRqqt*FBt/SRqqt_purity‚0.6165826916694641 ?&Purity/2FRt2SRqqt*FBt/event_proportion‚0.006417410913854837 jet/accuracy_3_of_5‚1 7 Purity/*FRt*SRqqt2FBt/FBt_purity‚0.9679478406906128 :"Purity/*FRt0SRqqt*FBt/event_purity‚0.20643983781337738 =&Purity/1FRt0SRqqt*FBt/event_proportion‚0.4530552327632904 >&Purity/1FRt0SRqqt1FBt/event_proportion‚0.00969587080180645 @&Purity/1FRt1SRqqt2FBt/event_proportion‚0.0002325148816453293 9"Purity/2FRt1SRqqt*FBt/SRqqt_purity‚0.7336450815200806 9"Purity/2FRt1SRqqt1FBt/SRqqt_purity‚0.7314814329147339 ?&Purity/*FRt*SRqqt2FBt/event_proportion‚0.010997953824698925 =&Purity/0FRt1SRqqt*FBt/event_proportion‚0.2203543484210968 >&Purity/0FRt2SRqqt0FBt/event_proportion‚0.02385602705180645 ,jet/accuracy_4_of_4‚0.013899499550461769 :"Purity/*FRt2SRqqt*FBt/event_purity‚0.17603075504302979 9"Purity/1FRt*SRqqt*FBt/SRqqt_purity‚0.7324050068855286 7 Purity/1FRt*SRqqt1FBt/FBt_purity‚0.9963945746421814 & Purity/1FRt2SRqqt*FBt/FBt_purity‚1 @&Purity/2FRt*SRqqt1FBt/event_proportion‚0.0016043527284637094 ?&Purity/2FRt1SRqqt0FBt/event_proportion‚0.011928013525903225 ("Purity/2FRt2SRqqt0FBt/event_purity‚0 ?&Purity/0FRt1SRqqt1FBt/event_proportion‚0.011555989272892475 ,&Purity/0FRt2SRqqt2FBt/event_proportion‚0 7 Purity/1FRt0SRqqt2FBt/FBt_purity‚0.9970238208770752 8 Purity/1FRt2SRqqt*FBt/FRt_purity‚0.04968917369842529 9"Purity/*FRt*SRqqt*FBt/SRqqt_purity‚0.6715810298919678 7 Purity/0FRt0SRqqt1FBt/FBt_purity‚0.9970456957817078 +jet/accuracy_3_of_3‚0.05896582454442978 particle/accuracy_1_of_2‚1 particle/accuracy_3_of_5‚1 :"Purity/*FRt1SRqqt2FBt/SRqqt_pI€’LJurity‚0.24074071645736694 9"Purity/0FRt2SRqqt*FBt/SRqqt_purity‚0.6146117448806763 7 Purity/2FRt*SRqqt1FBt/FRt_purity‚0.5233333706855774 @&Purity/2FRt0SRqqt1FBt/event_proportion‚0.0007672990905120969 ("Purity/2FRt2SRqqt*FBt/event_purity‚0 ,&Purity/2FRt2SRqqt2FBt/event_proportion‚0 *jet/accuracy_2_of_4‚0.7835207581520081 & Purity/2FRt0SRqqt1FBt/FBt_purity‚1 9"Purity/0FRt*SRqqt0FBt/SRqqt_purity‚0.6222657561302185 9"Purity/0FRt1SRqqt0FBt/event_purity‚0.6228646636009216 ?&Purity/2FRt2SRqqt0FBt/event_proportion‚0.006417410913854837 *jet/accuracy_1_of_2‚0.7738795280456543 jet/accuracy_5_of_5‚0 9"Purity/0FRt0SRqqt2FBt/event_purity‚0.9310600757598877 8 Purity/2FRt0SRqqt0FBt/FRt_purity‚0.12390178442001343 7 Purity/2FRt0SRqqt2FBt/FRt_purity‚0.7857142686843872 particle/accuracy_2_of_5‚1 particle/accuracy_5_of_5‚0 epoch‚8 9"Purity/0FRt2SRqqt0FBt/event_purity‚0.4053266942501068 8 Purity/1FRt*SRqqt0FBt/FRt_purity‚0.12806813418865204 7 Purity/1FRt*SRqqt2FBt/FRt_purity‚0.8577665686607361 9"Purity/1FRt0SRqqt1FBt/event_purity‚0.5190536379814148 &particle/f_score‚0.629570484161377O<ùñ >ƒ„| 8"Purity/0FRt2SRqqt0FBt/SRqqt_purity‚0.621488630771637 @&Purity/1FRt*SRqqt2FBt/event_proportion‚0.0027204242069274187 7 Purity/1FRt1SRqqt*FBt/FBt_purity‚0.9872215986251831 7 Purity/*FRt*SRqqt1FBt/FBt_purity‚0.9953683018684387 7 Purity/1FRt1SRqqt*FBt/FRt_purity‚0.1315348744392395 trainer/global_step‚3599 9"Purity/0FRt*SRqqt0FBt/event_purity‚0.5999293923377991 =&Purity/0FRt*SRqqt1FBt/event_proportion‚0.0436662957072258 =&Purity/0FRt0SRqqt0FBt/event_proportion‚0.0462472103536129 9"Purity/0FRt1SRqqt0FBt/SRqqt_purity‚0.6228646636009216 jet/accuracy_4_of_5‚0 8&Purity/1FRt2SRqqt0FBt/event_proportion‚ 0.02294921875 <&Purity/2FRt*SRqqt*FBt/event_proportion‚0.040062315762043 ("Purity/2FRt*SRqqt2FBt/SRqqt_purity‚0 <&Purity/*FRt*SRqqt0FBt/event_proportion‚0.881045401096344 7 Purity/*FRt0SRqqt0FBt/FRt_purity‚0.1350795179605484 7 Purity/*FRt0SRqqt2FBt/FRt_purity‚0.8643235564231873 ("Purity/0FRt0SRqqt0FBt/event_purity‚1 =&Purity/1FRt*SRqqt0FBt/event_proportion‚0.6105840802192688 9"Purity/*FRt*SRqqt2FBt/event_purity‚0.8186385035514832 particle/accuracy_1_of_3‚1 8 Purity/*FRt*SRqqt*FBt/FRt_purity‚0.14313514530658722 8 Purity/*FRt1SRqqt0FBt/FRt_purity‚0.12982788681983948 ) Purity/*FRt1SRqqt2FBt/FRt_purity‚0.75 :"Purity/1FRt0SRqqt*FBt/event_purity‚0.14746864140033722 & Purity/1FRt1SRqqt1FBt/FBt_purity‚1 ("Purity/2FRt1SRqqt0FBt/event_purity‚0 particle/accuracy_2_of_4‚1 6validation_average_jet_accuracy‚0.3389193117618561 7 Purity/*FRt*SRqqt*FBt/FBt_purity‚0.9882121086120605 =&Purity/*FRt1SRqqt0FBt/event_proportion‚0.3670479953289032 @&Purity/*FRt1SRqqt2FBt/event_proportion‚0.0006975446594879031 >&Purity/2FRt0SRqqt0FBt/event_proportion‚0.01990327425301075 7 Purity/*FRt1SRqqt1FBt/FBt_purity‚0.9936917424201965 9"Purity/1FRt*SRqqt2FBt/event_purity‚0.6751889586448669 7 Purity/1FRt0SRqqt1FBt/FRt_purity‚0.5823805332183838 8 Purity/1FRt1SRqqt0FBt/FRt_purity‚0.11597421020269394 7 Purity/1FRt1SRqqt2FBt/FRt_purity‚0.7777777910232544 A&Purity/2FRt0SRqqt2FBt/event_proportion‚0.00018601190822664648 9"Purity/*FRt2SRqqt0FBt/SRqqt_purity‚0.6993999481201172 0"Purity/0FRt2SRqqt1FBt/SRqqt_purity‚ 0.3984375 @&Purity/2FRt1SRqqt1FBt/event_proportion‚0.0008370535797439516 7 Purity/*FRt0SRqqt1FBt/FBt_purity‚0.9965531229972839 7 Purity/0FRt*SRqqt2FBt/FBt_purity‚0.9626487493515015 :"Purity/2FRt0SRqqt0FBt/event_purity‚0.01834227330982685 & Purity/2FRt*SRqqt2FBt/FBt_purity‚1 particle/accuracy_1_of_5‚1 =&Purity/*FRt0SRqqt1FBt/event_proportion‚0.0418526791036129 =&Purity/0FRt0SRqqt1FBt/event_proportion‚0.0313895083963871 ("Purity/0FRt2SRqqt1FBt/event_purity‚0 9"Purity/2FRt0SRqqt2FBt/event_purity‚0.5714285969734192 9"Purity/2FRt*SRqqt*FBt/SRqqt_purity‚0.7864935994148254 9"Purity/0FRt*SRqqt2FBt/SRqqt_purity‚0.1538461595773697 ("Purity/2FRt1SRqqt2FBt/event_purity‚0 ("Purity/*FRt2SRqqt1FBt/event_purity‚0 :"Purity/1FRt1SRqqt0FBt/event_purity‚0.03233988955616951 9"Purity/1FRt1SRqqt2FBt/SRqqt_purity‚0.3333333432674408 9"Purity/*FRt1SRqqt0FBt/SRqqt_purity‚0.6714556813240051 8 Purity/*FRt2SRqqt0FBt/FRt_purity‚0.04693333059549332 A&Purity/1FRt2SRqqt1FBt/event_proportion‚0.00027901786961592734 +jet/accuracy_3_of_4‚0.07620833069086075 7 Purity/*FRt*SRqqt1FBt/FRt_purity‚0.5190966129302979 :"Purity/*FRt1SRqqt*FBt/event_purity‚0.36532261967658997 :"Purity/*FRt2SRqqt1FBt/SRqqt_purity‚0.35879629850387573 ?&Purity/0FRt0SRqqt2FBt/event_proportion‚0.007626487873494625 =&Purity/2FRt1SRqqt*FBt/event_proportion‚0.0127883180975914 particle/accuracy_1_of_1‚1 % _timestamp‚1.7844628107683563e+09 7 Purity/*FRt2SRqqt1FBt/FBt_purity‚0.9814814329147339 7 Purity/1FRt0SRqqt*FBt/FBt_purity‚0.9941900372505188 :"Purity/1FRt0SRqqt0FBt/event_purity‚0.13610389828681946 9"Purity/2FRt*SRqqt0FBt/SRqqt_purity‚0.7900545001029968 9"Purity/0FRt*SRqqt1FBt/event_purity‚0.8358891606330872 8 Purity/1FRt0SRqqt*FBt/FRt_purity‚0.14962711930274963 =&Purity/*FRt0SRqqt*FBt/event_proportion‚0.5129278302192688 =&Purity/0FRt0SRqqt*FBt/event_proportion‚0.0390159972012043 , Purity/0FRt2SRqqt1FBt/FBt_purity‚0.96875 9"Purity/1FRt1SRqqt*FBt/SRqqt_purity‚0.7303345799446106 *jet/accuracy_2_of_3‚0.5495484471321106 9"Purity/*FRt*SRqqt0FBt/SRqqt_purity‚0.6776919364929199 9"Purity/0FRt*SRqqt1FBt/SRqqt_purity‚0.5051735639572144 ("Purity/1FRt2SRqqt*FBt/event_purity‚0 =&Purity/*FRt*SRqqt*FBt/event_proportion‚0.9537528157234192 >&Purity/1FRt2SRqqt*FBt/event_proportion‚0.02322823740541935 8 Purity/1FRt2SRqqt0FBt/FRt_purity‚0.05019146949052811 9"Purity/2FRt1SRqqt0FBt/SRqqt_purity‚0.7365633845329285 *jet/accuracy_1_of_3‚0.9184180498123169 'particle/accuracy‚0.840839684009552 :"Purity/*FRt*SRqqt2FBt/SRqqt_purity‚0.24074071645736694 =&Purity/*FRt1SRqqt1FBt/event_proportion‚0.0188569575548172 9"Purity/*FRt1SRqqt1FBt/SRqqt_purity‚0.5697539448738098 =&Purity/0FRt*SRqqt0FBt/event_proportion‚0.2322126179933548 8"Purity/0FRt1SRqqt1FBt/SRqqt_purity‚0.522489070892334 =&Purity/1FRt*SRqqt*FBt/event_proportion‚0.6297432780265808 9"Purity/1FRt0SRqqt2FBt/event_purity‚0.7392479777336121 +particle/sensitivity‚0.6298382878303528 9"Purity/*FRt1SRqqt1FBt/event_purity‚0.3556191921234131 :"Purity/0FRt1SRqqt1FBt/event_purity‚0.45889630913734436 :"Purity/2FRt0SRqqt*FBt/event_purity‚0.03457769751548767 particle/accuracy_1_of_4‚1 jet/accuracy_2_of_5‚1 _step‚296 :"Purity/*FRt*SRqqt*FBt/event_purity‚0.26909762620925903 7 Purity/0FRt0SRqqt*FBt/FBt_purity‚0.9874101281166077 :"Purity/1FRt*SRqqt1FBt/event_purity‚0.39110028743743896 9"Purity/1FRt*SRqqt2FBt/SRqqt_purity‚0.3333333432674408 =&Purity/1FRt1SRqqt*FBt/event_proportion‚0.1534598171710968 8 Purity/2FRt0SRqqt*FBt/FRt_purity‚0.14758087694644928 7 Purity/2FRt1SRqqt1FBt/FRt_purity‚0.4490741193294525 =&Purity/*FRt0SRqqt0FBt/event_proportion‚0.4607747495174408 :"Purity/0FRt2SRqqt*FBt/event_purity‚0.39385467767715454 >&Purity/0FRt2SRqqt*FBt/event_proportion‚0.02457682229578495 9"Purity/2FRt2SRqqt*FBt/SRqqt_purity‚0.8465028405189514 & Purity/2FRt0SRqqt*FBt/FBt_purity‚1 =&Purity/*FRt*SRqqt1FBt/event_proportion‚0.0617094486951828 9"Purity/*FRt2SRqqt*FBt/SRqqt_purity‚0.6931633949279785 :"Purity/1FRt*SRqqt*FBt/event_purity‚0.11588989198207855 7 Purity/1FRt*SRqqt*FBt/FBt_purity‚0.9918419718742371 9"Purity/1FRt*SRqqt1FBt/SRqqt_purity‚0.6224424839019775 & Purity/1FRt2SRqqt1FBt/FBt_purity‚1 (jet/accuracy_1_of_1‚0.28925821185112 :"Purity/*FRt2SRqqt0FBt/event_purity‚0.17934942245483398 8 Purity/1FRt*SRqqt*FBt/FRt_purity‚0.14154718816280365 =&Purity/1FRt1SRqqt0FBt/event_proportion‚0.1467633992433548 *jet/accuracy_1_of_4‚0.9684416651725769 particle/accuracy_2_of_3‚1 9"Purity/*FRt1SRqqt0FBt/event_purity‚0.3666115403175354 9"Purity/0FRt*SRqqt2FBt/event_purity‚0.8772681951522827 ("Purity/2FRt1SRqqt*FBt/event_purity‚0 ("Purity/2FRt1SRqqt2FBt/SRqqt_purity‚0 <&Purity/*FRt2SRqqt*FBt/event_proportion‚0.054222472012043 ("Purity/1FRt1SRqqt2FBt/event_purity‚0 7 Purity/1FRt1SRqqt2FBt/FBt_purity‚0.7777777910232544 =&Purity/2FRt*SRqqt0FBt/event_proportion‚0.0382486991584301 9"Purity/2FRt*SRqqt1FBt/SRqqt_purity‚0.7314814329147339 9 Purity/2FRt2SRqqt0FBt/FRt_purity‚0.036971911787986755 =&Purity/*FRt1SRqqt*FBt/event_proportion‚0.3866024911403656 9"Purity/0FRt1SRqqt*FBt/event_purity‚0.6128073930740356 9"Purity/1FRt1SRqqt0FBt/SRqqt_purity‚0.7352995872497559 8 Purity/2FRt*SRqqt0FBt/FRt_purity‚0.13830778002738953 :"Purity/2FRt*SRqqt1FBt/event_purity‚0.16500000655651093 /particle/accuracy_4_of_4‚0.9759465456008911 ) Purity/2FRt*SRqqt2FBt/FRt_purity‚0.75 & Purity/2FRt1SRqqt*FBt/FBt_purity‚1 :"Purity/*FRt*SRqqt0FBt/event_purity‚0.23201200366020203 7 Purity/*FRt0SRqqt2FBt/FBt_purity‚0.9742013812065125 ,&Purity/*FRt2SRqqt2FBt/event_proportion‚0 5 Purity/0FRt*SRqqt1FBt/FBt_purity‚0.99461829662323 & Purity/2FRt*SRqqt1FBt/FBt_purity‚1 8 Purity/2FRt1SRqqt*FBt/FRt_purity‚0.22816292941570282 7 Purity/2FRt0SRqqt1FBt/FRt_purity‚0.6197916269302368 @&Purity/*FRt2SRqqt1FBt/event_proportion‚0.0009998140158131719 9"Purity/0FRt*SRqqt*FBt/SRqqt_purity‚0.6158679127693176 7 Purity/1FRt*SRqqt1FBt/FRt_purity‚0.5208590626716614 @&Purity/1FRt1SRqqt1FBt/event_proportion‚0.0064639137126505375 ;"Purity/2FRt*SRqqt0FBt/event_purity‚0.009579349309206009 7 Purity/*FRt1SRqqt2FBt/FBt_purity‚0.8518518209457397 6 Purity/0FRt1SRqqt*FBt/FBt_purity‚0.984364926815033 =&Purity/0FRt1SRqqt0FBt/event_proportion‚0.2083565890789032 @&Purity/0FRt1SRqqt2FBt/event_proportion‚0.0004417782765813172 =&Purity/2FRt0SRqqt*FBt/event_proportion‚0.0208565853536129 9"Purity/0FRt*SRqqt*FBt/event_purity‚0.6443859934806824 ("Purity/1FRt2SRqqt0FBt/event_purity‚0 & Purity/1FRt2SRqqt1FBt/FRt_purity‚0 9 Purity/2FRt2SRqqt*FBt/FRt_purity‚0.036971911787986755 7 Purity/0FRt1SRqqt1FBt/FBt_purity‚0.9900330901145935 @&Purity/0FRt2SRqqt1FBt/event_proportion‚0.0007207961170934141 9"Purity/1FRt*SRqqt0FBt/SRqqt_purity‚0.7375082969665527 /particle/accuracy_2_of_2‚0.9998962879180908 ("Purity/0FRt1SRqqt2FBt/event_purity‚0 *loss/val_total_loss‚11.003473853957328 =&Purity/*FRt0SRqqt2FBt/event_proportion‚0.0103004090487957 8 Purity/2FRt*SRqqt*FBt/FRt_purity‚0.15644004940986633 & Purity/2FRt1SRqqt1FBt/FBt_purity‚1 +jet/accuracy_2_of_2‚0.13320466876029968 6 Purity/0FRt*SRqqt*FBt/FBt_purity‚0.986178994178772 & Purity/2FRt*SRqqt*FBt/FBt_purity‚1 8 Purity/2FRt1SRqqt0FBt/FRt_purity‚0.21442243456840515 ("Purity/2FRt1SRqqt1FBt/event_purity‚0 +particle/specificity‚0.8985463976860046 ( Purity/2FRt1SRqqt2FBt/FRt_purity‚0.5 9"Purity/0FRt1SRqqt*FBt/SRqqt_purity‚0.6165826916694641 ?&Purity/2FRt2SRqqt*FBt/event_proportion‚0.006417410913854837 jet/accuracy_3_of_5‚1 7 Purity/*FRt*SRqqt2FBt/FBt_purity‚0.9679478406906128 :"Purity/*FRt0SRqqt*FBt/event_purity‚0.20643983781337738 =&Purity/1FRt0SRqqt*FBt/event_proportion‚0.4530552327632904 >&Purity/1FRt0SRqqt1FBt/event_proportion‚0.00969587080180645 @&Purity/1FRt1SRqqt2FBt/event_proportion‚0.0002325148816453293 9"Purity/2FRt1SRqqt*FBt/SRqqt_purity‚0.7336450815200806 9"Purity/2FRt1SRqqt1FBt/SRqqt_purity‚0.7314814329147339 ?&Purity/*FRt*SRqqt2FBt/event_proportion‚0.010997953824698925 =&Purity/0FRt1SRqqt*FBt/event_proportion‚0.2203543484210968 >&Purity/0FRt2SRqqt0FBt/event_proportion‚0.02385602705180645 ,jet/accuracy_4_of_4‚0.013899499550461769 :"Purity/*FRt2SRqqt*FBt/event_purity‚0.17603075504302979 9"Purity/1FRt*SRqqt*FBt/SRqqt_purity‚0.7324050068855286 7 Purity/1FRt*SRqqt1FBt/FBt_purity‚0.9963945746421814 & Purity/1FRt2SRqqt*FBt/FBt_purity‚1 @&Purity/2FRt*SRqqt1FBt/event_proportion‚0.0016043527284637094 ?&Purity/2FRt1SRqqt0FBt/event_proportion‚0.011928013525903225 ("Purity/2FRt2SRqqt0FBt/event_purity‚0 ?&Purity/0FRt1SRqqt1FBt/event_proportion‚0.011555989272892475 ,&Purity/0FRt2SRqqt2FBt/event_proportion‚0 7 Purity/1FRt0SRqqt2FBt/FBt_purity‚0.9970238208770752 8 Purity/1FRt2SRqqt*FBt/FRt_purity‚0.04968917369842529 9"Purity/*FRt*SRqqt*FBt/SRqqt_purity‚0.6715810298919678 7 Purity/0FRt0SRqqt1FBt/FBt_purity‚0.9970456957817078 +jet/accuracy_3_of_3‚0.05896582454442978 particle/accuracy_1_of_2‚1 particle/accuracy_3_of_5‚1 :"Purity/*FRt1SRqqt2FBt/SRqqt_purity‚0.24074071645736694 9"Purity/0FRt2SRqqt*FBt/SRqqt_purity‚0.6146117448806763 7 Purity/2FRt*SRqqt1FBt/FRt_purity‚0.5233333706855774 @&Purity/2FRt0SRqqt1FBt/event_proportion‚0.0007672990905120969 ("Purity/2FRt2SRqqt*FBt/event_purity‚0 ,&Purity/2FRt2SRqqt2FBt/event_proportion‚0 *jet/accuracy_2_of_4‚0.7835207581520081 & Purity/2FRt0SRqqt1FBt/FBt_purity‚1 9"Purity/0FRt*SRqqt0FBt/SRqqt_purity‚0.6222657561302185 9"Purity/0FRt1SRqqt0FBt/event_purity‚0.6228646636009216 ?&Purity/2FRt2SRqqt0FBt/event_proportion‚0.006417410913854837 *jet/accuracy_1_of_2‚0.7738795280456543 jet/accuracy_5_of_5‚0 9"Purity/0FRt0SRqqt2FBt/event_purity‚0.9310600757598877 8 Purity/2FRt0SRqqt0FBt/FRt_purity‚0.12390178442001343 7 Purity/2FRt0SRqqt2FBt/FRt_purity‚0.7857142686843872 particle/accuracy_2_of_5‚1 particle/accuracy_5_of_5‚0 epoch‚8 9"Purity/0FRt2SRqqt0FBt/event_purity‚0.4053266942501068 8 Purity/1FRt*SRqqt0FBt/FRt_purity‚0.12806813418865204 7 Purity/1FRt*SRqqt2FBt/FRt_purity‚0.8577665686607361 9"Purity/1FRt0SRqqt1FBt/event_purity‚0.5190536379814148 &particle/f_score‚0.629570484161377 :"Purity/*FRt0SRqqt0FBt/event_purity‚0.13101819157600403 =&Purity/0FRt*SRqqt*FBt/event_proportion‚0.2839471697807312 =&Purity/1FRt0SRqqt0FBt/event_proportion‚0.4408714771270752 /particle/accuracy_3_of_3‚0.9994933605194092 *"Purity/2FRt*SRqqt2FBt/event_purity‚0.5 7 Purity/*FRt0SRqqt1FBt/FRt_purity‚0.5833929181098938 , Purity/0FRt2SRqqt*FBt/FBt_purity‚0.96875 :"Purity/1FRt1SRqqt*FBt/event_purity‚0.04017571732401848 & Purity/2FRt0SRqqt2FBt/FBt_purity‚1 7 Purity/0FRt0SRqqt2FBt/FBt_purity‚0.9655299186706543 9"Purity/1FRt2SRqqt*FBt/SRqqt_purity‚0.7388749718666077 /particle/accuracy_3_of_4‚0.9965986609458923 9"Purity/0FRt1SRqqt2FBt/SRqqt_purity‚0.1538461595773697 9"Purity/2FRt0SRqqt1FBt/event_purity‚0.3489583134651184 7 Purity/*FRt1SRqqt1FBt/FRt_purity‚0.4491068124771118 7 Purity/*FRt2SRqqt*FBt/FBt_purity‚0.9814814329147339 7 Purity/1FRt0SRqqt1FBt/FBt_purity‚0.9928571581840515 9"Purity/1FRt2SRqqt0FBt/SRqqt_purity‚0.7444075345993042 9"Purity/*FRt0SRqqt2FBt/event_purity‚0.8757578730583191 8 Purity/*FRt2SRqqt*FBt/FRt_purity‚0.04660569131374359 :"Purity/1FRt*SRqqt0FBt/event_purity‚0.10606343299150467 7 Purity/1FRt*SRqqt2FBt/FBt_purity‚0.9815192222595215 ,&Purity/1FRt2SRqqt2FBt/event_proportion‚0 particle/accuracy_4_of_5‚1 _runtime‚2485.157216239 9"Purity/*FRt*SRqqt1FBt/SRqqt_purity‚0.5486062169075012 9"Purity/*FRt0SRqqt1FBt/event_purity‚0.8739169239997864 ?&Purity/0FRt*SRqqt2FBt/event_proportion‚0.008068266324698925 8 Purity/1FRt0SRqqt0FBt/FRt_purity‚0.13610389828681946 7 Purity/1FRt0SRqqt2FBt/FRt_purity‚0.8681973218917847 :"Purity/1FRt1SRqqt1FBt/event_purity‚0.21879377961158752 8 Purity/1FRt1SRqqt1FBt/FRt_purity‚0.44352903962135315 9"Purity/*FRt1SRqqt*FBt/SRqqt_purity‚0.6655905842781067 >&Purity/1FRt*SRqqt1FBt/event_proportion‚0.01643880270421505 ,&Purity/2FRt2SRqqt1FBt/event_proportion‚0 9"Purity/*FRt*SRqqt1FBt/event_purity‚0.6999806761741638 8&Purity/*FRt2SRqqt0FBt/event_proportion‚ 0.05322265625 @&Purity/1FRt0SRqqt2FBt/event_proportion‚0.0024879092816263437 8"Purity/1FRt1SRqqt1FBt/SRqqt_purity‚0.641708254814148 :"Purity/2FRt*SRqqt*FBt/event_purity‚0.01799742318689823 @&Purity/2FRt*SRqqt2FBt/event_proportion‚0.0002092633949359879 7 Purity/*FRt0SRqqt*FBt/FBt_purity‚0.9894615411758423 9"Purity/0FRt0SRqqt1FBt/event_purity‚0.9970456957817078 :"Purity/1FRt2SRqqt1FBt/SRqqt_purity‚0.32499998807907104 8 Purity/*FRt0SRqqt*FBt/FRt_purity‚0.14942173659801483 @&Purity/2FRt1SRqqt2FBt/event_proportion‚2.325148852833081e-05 & Purity/2FRt1SRqqt2FBt/FBt_purity‚1 jet/accuracy_1_of_5‚1 6 Purity/*FRt1SRqqt*FBt/FBt_purity‚0.984939694404602 & Purity/*FRt2SRqqt1FBt/FRt_purity‚0 ("Purity/1FRt2SRqqt1FBt/event_purity‚0 9"Purity/2FRt2SRqqt0FBt/SRqqt_purity‚0.8465028405189514 7 Purity/0FRt1SRqqt2FBt/FBt_purity‚0.9230769276618958 7 Purity/*FRt*SRqqt0FBt/FRt_purity‚0.1291438639163971 7 Purity/*FRt*SRqqt2FBt/FRt_purity‚0.8512850403785706 8 Purity/*FRt1SRqqt*FBt/FRt_purity‚0.14545424282550812 ("Purity/*FRt1SRqqt2FBt/event_purity‚0 9"Purity/0FRt0SRqqt*FBt/event_purity‚0.9849008917808533¨*Ôæ@Ÿ„‚(:” ßûòÒš¤ñ¤ gpu.0.gpu‚98.0 gpu.0.memory‚23, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚47.0 gpu.0.powerWatts‚113.439' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚37.812999999999995 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0êZ¬}m…‚(:â ßûòÒöج¦ proc.cpu.threads‚17& disk.nvme1c1n1.out‚15451.501953125 disk.nvme1n1.in‚ 129.765625$ disk.nvme0n1.out‚29987.009765625# proc.memory.rssMB‚ 2965.67578125( disk./.usagePercent‚76.2729060240278% disk./.usageGB‚1170.9797897338867 disk.nvme1c1n1.in‚ 129.765625$ disk.nvme1n1.out‚15451.501953125 disk.md0.out‚ 61.53515625 disk.dm-0.in‚ 1982.2109375 disk.dm-1.out‚1653.470703125& disk.nvme0c0n1.out‚29987.009765625 network.sent‚ 5.323248e+06 network.recv‚675666 disk.rbd0.in‚0 disk.rbd0.out‚0" disk.nvme0c0n1.in‚ 1852.4453125 disk.nvme0n1.in‚ 1852.4453125* proc.memory.percent‚18.101048469543457 cpu‚25.847269590858318 disk.md0.in‚0! disk.dm-0.out‚43652.595703125 disk.dm-1.in‚0$ memory_percent‚99.20229911804199' proc.memory.availableMB‚ 130.6953125âŠÃÛž†‚(:“ îûòÒ¶’̤ gpu.0.gpu‚91.0 gpu.0.memory‚17, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚48.0 gpu.0.powerWatts‚213.969' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚71.32300000000001 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚09prq‡‚(:æ îûòÒàî¤# proc.memory.rssMB‚ 2965.67578125 network.sent‚ 5.382221e+06 disk.nvme0n1.in‚ 1852.4453125' disk.nvme1c1n1.out‚15476.3330078125 cpu‚25.84819752106688 proc.cpu.threads‚17) disk./.usagePercent‚76.27290950266395 disk.dm-1.in‚0 disk.rbd0.out‚0 disk.nvme1c1n1.in‚ 129.765625* proc.memory.percent‚18.101048469543457 network.recv‚679734 disk.dm-0.in‚ 1982.2109375! disk.dm-1.out‚1677.6884765625& disk.nvme0c0n1.out‚29989.771484375% disk.nvme1n1.out‚15476.3330078125 disk.md0.in‚0 disk.md0.out‚ 61.88671875$ memory_percent‚99.20423030853271% disk./.usageGB‚1170.9798431396484! disk.dm-0.out‚43655.212890625 disk.rbd0.in‚0" disk.nvme0c0n1.in‚ 1852.4453125$ disk.nvme0n1.out‚29989.771484375 disk.nvme1n1.in‚ 129.765625( proc.memory.availableMB‚ 130.37890625%Í U2ˆ- _wandbruntime‚2519  _runtime‚2519áwkSɉà ' lr-AdamW/pg2‚0.00013001343739749933 trainer/global_step‚3649 % _timestamp‚1.7844628450568612e+09 _runtime‚2519.445987246 _step‚297 ' lr-AdamW/pg1‚0.00013001343739749933í¼âÎŠÈ trainer/global_step‚3649 % _timestamp‚1.7844628450568612e+09 _runtime‚2519.445987246 _step‚297 ' lr-AdamW/pg1‚0.00013001343739749933 ' lr-AdamW/pg2‚0.00013001343739749933©Qº¹2‹- _wandbruntime‚2519  _runtime‚2519}pïà'ŒÚO TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 trainer/global_step‚3649 _step‚298 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚286295908544000 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 _runtime‚2519.446194097 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚28946021 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚28946021 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚19429992 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 208873472 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚279940058666800 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 % _timestamp‚1.7844628450577161e+09 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚19429996 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚8275437 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚283283853910528 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚283283836871168 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3101365616128 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚27705262 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚8275270 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚286296014243328 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3012160332800 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚282943957696248 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚212178252135936 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚282943853870462 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚279940041627440 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚215279711861760 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚212178346245632 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚65674056 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚36728033 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3003812243022 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚27705433 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3012071672832 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚171 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 114763776 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3003899029448 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚286296014243328 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚36717890 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚286295908544000 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚36728033 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚215279711861760 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚212178346245632 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚65674056 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 TGDeviceStatsMonitor-|ï(À.on_train_batch_start/allocation.small_pool.allocated‚36728033 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3003812243022 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚27705433 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3012071672832 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚171 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 114763776 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3003899029448 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚286296014243328 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚36717890 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚286295908544000 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚36728033 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚283283853910528 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚283283836871168 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3101365616128 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚27705262 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚8275270 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚286296014243328 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3012160332800 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚282943957696248 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚212178252135936ªÍ#À2Ž- _wandbruntime‚2519  _runtime‚2519Aw®ê&äM ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚283359003576832 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 _step‚299 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚65681360 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚283359003576832 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚283018927890454 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚65691519 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚36737867 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3012946055680 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚283358986537472 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚215336446624256 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 trainer/global_step‚3649 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚65681360 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚286371949632512 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚180 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚27712422 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚280014332152832 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 _runtime‚2519.868620394 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8277374 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚28953650 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚212234483973632 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3102177808384 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3004595737622 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚280014349192192 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3004682524160 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 % _timestamp‚1.7844628454799485e+09 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 215157760 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚28953652 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚36727710 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚28953650 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚286371949632512 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3012857388544 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚212234389863936 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚286371843926016 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚27712606 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157Ý Y[ï&éM JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚27712422 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚280014332152832 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 _runtime‚2519.868620394 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8277374 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚28953650 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚212234483973632 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3102177808384 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3004595737622 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚280014349192192 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3004682524160 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 % _timestamp‚1.7844628454799485e+09 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 215157760 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚28953652 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚36727710 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚28953650 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚286371949632512 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3012857388544 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚212234389863936 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚286371843926016 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚27712606 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚283359003576832 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 _step‚299 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚65681360 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚283359003576832 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚283018927890454 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚65691519 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚36737867 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3012946055680 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚283358986537472 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚215336446624256 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 trainer/global_step‚3649 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚65681360 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚286371949632512 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚180 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 EÎœÈ ' lr-AdamW/pg2‚0.00012973388853634228 trainer/global_step‚3699 % _timestamp‚1.7844628793706105e+09 _runtime‚2553.759799971 _step‚301 ' lr-AdamW/pg1‚0.00012973388853634228­[aÓ2- _wandbruntime‚2553  _runtime‚2553~JÞß'žÙO E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚287041337225728 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚287041337225728 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3051421613056 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚8384217 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3051421613056 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 129443840 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3141869685760 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚193 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚29327471 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚286697722496656 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚29327473 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚290092741799424 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚283654584936400 _step‚302 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚37220885 $ _timestamp‚1.784462879371353e+09 _runtime‚2553.759892531 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 trainer/global_step‚3699 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚28066820 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚66538213 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚8384028 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚66548358 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚19682792 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚218127008221696 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚290092741799424 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3051510273024 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚189 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 150006272 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚214985138535936 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚37210742 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚287041320186368µ·¥Dä'ŸÞO ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚287041337225728 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3051421613056 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚8384217 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3051421613056 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 129443840 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3141869685760 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚193 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚29327471 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚286697722496656 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚29327473 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚290092741799424 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚283654584936400 _step‚302 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚37220885 $ _timestamp‚1.784462879371353e+09 _runtime‚2553.759892531 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 trainer/global_step‚3699 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚28066820 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚66538213 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚8384028 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚66548358 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚19682792 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚218127008221696 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚290092741799424 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3051510273024 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚189 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 150006272 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚214985138535936 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚37210742 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚287041320186368 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚287041337225728®/¥¼ 2 - _wandbruntime‚2554  _runtime‚2554ë~/ê&¡äM J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚29335100 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚290168782887936 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8386216 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚283728875461792 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚283728858422432 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚19687848 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚178 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3142805407232 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚19687852 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚286772796516760 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚66565821 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚215041370373632 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚28074064 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 150006272 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 trainer/global_step‚3699 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚290168782887936 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚286772692690862 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3043834268430 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚66565821 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚29335102 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚287116486892032 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚218184175780864 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚37230719 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚218183958525952 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚215041276263936 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚37220562 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚37220562 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚29335102 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175í0E¢éM B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3052295995904 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚29335100 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 trainer/global_step‚3699 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚290168782887936 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 K;DeviceStatsMonitor.on_traiyºèzª"n_batch_end/active_bytes.all.peak‚ 12301877248 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚286772692690862 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3043834268430 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚66565821 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚29335102 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚287116486892032 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚218184175780864 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚37230719 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚218183958525952 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚215041276263936 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚37220562 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚37220562 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚29335102 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚29335100 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚290168782887936 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8386216 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚283728875461792 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚283728858422432 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚19687848 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚178 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3142805407232 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚19687852 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚286772796516760 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚66565821 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚215041370373632 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚28074064 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 150006272 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚293889680754176 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚287369111206000 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 133638144 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3182495797248 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _runtime‚2588.154421099 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚29708921 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚37713737 _step‚306 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚37713737 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚290798803501568 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚29708923 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3082376091064 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚290798820540928 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚67422660 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚19935592 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚193 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚189 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚217792024935936 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚293889575054848 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3090771553280 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚290798803501568 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚293889575054848 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 227747840 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200š¸nÁ"¯ÞO M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚67422660 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚19935592 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚193 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚189 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚217792024935936 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚293889575054848 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3090771553280 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚290798803501568 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚293889575054848 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 227747840 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚293889680754176 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚287369111206000 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 133638144 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3182495797248 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 _runtime‚2588.154421099 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚29708921 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚37713737 _step‚306 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚37713‰-¿n#737 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚290798803501568 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚29708923 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3082376091064 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚290798820540928²Œøüÿ2°- _wandbruntime‚2588  _runtime‚2588ý°@¢é&±ãM H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3091645936128 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚67440123 _runtime‚ 2588.57506137 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚28435324 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚217848256773632 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 150006272 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚287443401731392 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3083072799238 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚217848162663936 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚29716552 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚290526457491270 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚29716550 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 217254912 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚221031470849024 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _step‚307 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚290873953167872 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚67429964 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚290526561317168 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚37723571 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚37713414 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3083159585776 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 123145216 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3183308185088 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚290873970207232 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚19940652 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚287443384692032 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚180 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚293965510436864 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚37713414 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚8494672 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚221031688103936 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚290873953167872 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚28435144 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3091557268992 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚37723571 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682P½>î&²èM ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚217848162663936 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚29716552 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚290526457491270 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚29716550 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 217254912 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚221031470849024 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _step‚307 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚290873953167872 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚67429964 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚290526561317168 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚37723571 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚37713414 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3083159585776 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 123145216 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3183308185088 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚290873970207232 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚19940652 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚287443384692032 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚180 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚293965510436864 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚37713414 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚8494672 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚221031688103936 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚290873953167872 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚28435144 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3091557268992 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚37723571 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3091645936128 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚67440123 _runtime‚ 2588.57506137 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚28435324 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚217848256773632 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 150006272 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚287443401731392 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3083072799238³øeµ2³- _wandbruntime‚2588  _runtime‚2588^tTA´ÿ /loss/FRt1/assignment_loss‚2.588500489705038 2loss/SRqqt1/assignment_loss‚0.8311257247943273 /loss/FRt1/detection_loss‚0.5905919981823485 /loss/FRt2/detection_loss‚1.3959545813847996 epoch‚9 _step‚308 1loss/FBt1/assignment_loss‚0.11320926934685475 % _timestamp‚1.7844629141875768e+09 _runtime‚2588.578811767 1loss/SRqqt1/detection_loss‚1.0711853367323845 1loss/SRqqt2/detection_loss‚0.7229047303065381 .loss/FBt2/detection_loss‚1.417312292713258 trainer/global_step‚3749 0loss/FRt2/assignment_loss‚1.2062654023642068 1loss/SRqqt2/assignment_loss‚0.670032522441601 2loss/FBt2/assignment_loss‚0.016158895325325297 /loss/FBt1/detection_loss‚0.5575593427987077 %loss/total_loss‚11.18080058609539P¿ª1 µ„ trainer/global_step‚3749 0loss/FRt2/assignment_loss‚1.2062654023642068 1loss/SRqqt2/assignment_loss‚0.670032522441601 2loss/FBt2/assignment_loss‚0.016158895325325297 /loss/FBt1/detection_loss‚0.5575593427987077 %loss/total_loss‚11.18080058609539 /loss/FRt1/assignment_loss‚2.588500489705038 2loss/SRqqt1/assignment_loss‚0.8311257247943273 /loss/FRt1/detection_loss‚0.5905919981823485 /loss/FRt2/detection_loss‚1.3959545813847996 epoch‚9 _step‚308 1loss/FBt1/assignment_loss‚0.11320926934685475 % _timestamp‚1.7844629141875768e+09 _runtime‚2588.578811767 1loss/SRqqt1/detection_loss‚1.0711853367323845 1loss/SRqqt2/detection_loss‚0.7229047303065381 .loss/FBt2/detection_loss‚1.417312292713258´è® r¶‚(:ç ÈüòÒªñɤ disk.dm-1.in‚0 disk.nvme1n1.in‚ 129.765625& disk.nvme1n1.out‚15521.02783203125 disk.md0.in‚0! proc.memory.availableMB‚65.75 disk.nvme0n1.in‚ 1852.4453125 disk.nvme1c1n1.in‚ 129.765625( disk.nvme1c1n1.out‚15521.02783203125# proc.memory.rssMB‚ 2965.67578125* proc.memory.percent‚18.101048469543457 disk.rbd0.out‚0" disk.nvme0c0n1.in‚ 1852.4453125 network.recv‚702187) disk./.usagePercent‚76.27296217915446% disk./.usageGB‚1170.9806518554688# disk.dm-0.out‚43667.25439453125" disk.dm-1.out‚1718.16162109375 disk.rbd0.in‚0 disk.dm-0.in‚ 1982.2109375( disk.nvme0c0n1.out‚30002.13623046875 disk.md0.out‚ 63.97265625$ memory_percent‚99.59869384765625 cpu‚25.8507574627531 proc.cpu.threads‚17& disk.nvme0n1.out‚30002.13623046875 network.sent‚ 5.525833e+06+Mxºž·‚(:“ ÈüòÒœ˜ª¥ gpu.0.gpu‚45.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚47.0 gpu.0.powerWatts‚113.755' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚37.91833333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Ðu;q¸‚(:æ ×üòÒù¤ñ¤* proc.memory.percent‚18.101048469543457 proc.cpu.threads‚17 disk.dm-1.in‚0 disk.rbd0.in‚0& disk.nvme1n1.out‚15522.00439453125 network.sent‚ 5.589094e+06 network.recv‚706013 disk.md0.in‚0" disk.nvme0c0n1.in‚ 1852.4453125% disk.nvme0n1.out‚30005.3896484375 disk.nvme1c1n1.in‚ 129.765625$ proc.memory.availableMB‚65.46875 cpu‚25.85370089101829% disk./.usageGB‚1170.9814338684082 disk.dm-0.in‚ 1982.2109375" disk.dm-1.out‚1718.18115234375 disk.rbd0.out‚0' disk.nvme0c0n1.out‚30005.3896484375 disk.nvme0n1.in‚ 1852.4453125 disk.nvme1n1.in‚ 129.765625$ memory_percent‚99.60041046142578) disk./.usagePercent‚76.27301311632687 disk.md0.out‚ 64.34765625 disk.dm-0.out‚43670.68359375( disk.nvme1c1n1.out‚15522.00439453125# proc.memory.rssMB‚ 2965.67578125YC(’›¹‚(: ×üòÒÊø¢¥ gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚48.0 gpu.0.powerWatts‚198.38' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚66.12666666666667 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Mwý,2º- _wandbruntime‚2622  _runtime‚2622ĻŠÉ»Ã _runtime‚2622.953456775 _step‚309 ' lr-AdamW/pg1‚0.00012916322625830794 ' lr-AdamW/pg2‚0.00012916322625830794 trainer/global_step‚3799 % _timestamp‚1.7844629485650344e+09„@HÎ¼È ' lr-AdamW/pg1‚0.00012916322625830794 ' lr-AdamW/pg2‚0.00012916322625830794 trainer/global_step‚3799 % _timestamp‚1.7844629485650344e+09 _runtime‚2622.953456775 _step‚309µ¶¶—62½- _wandbruntime‚2622  _runtime‚2622†0mñ ¾ÚO UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚297686408310272 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚30090373 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚178 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚20188396 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3223211252736 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚8601019 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚294205252097472 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚20188392 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚220599005445632 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚291083637475600 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 118958080 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚38206589 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚38206589 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚297686514009600 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 213067776 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚297686514009600 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3223092294656 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚30090373 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚28789415 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚174 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚294556286816768 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚38196446 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3130121493504 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejectio’z¯Ô ns‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚30090371 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚294205148271686 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3121614621872 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 _runtime‚2622.954809437 _step‚310 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚294556303856128 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3130210153472 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚38196446 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚291083620436240 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3121527835446 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚30090371 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚297686408310272 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚294556303856128 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 150006272 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 % _timestamp‚1.7844629485662227e+09 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚68296962 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812B 3zå'¿ßO QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚174 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚294556286816768 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚38196446 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3130121493504 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚30090371 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚294205148271686 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3121614621872 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 _runtime‚2622.954809437 _step‚310 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚294556303856128 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3130210153472 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚38196446 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚291083620436240 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3121527835446 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚30090371 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚297686408310272 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚294556303856128 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 150006272 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 % _timestamp‚1.7844629485662227e+09 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚68296962 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚297686408310272 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚30090373 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚178 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚20188396 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3223211252736 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚8601019 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚294205252097472 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚20188392 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚220599005445632 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚291083637475600 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 118958080 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚38206589 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚38206589 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚297686514009600 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 213067776 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚297686514009600 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3223092294656 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚30090373 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚28789415¶Pf'2À- _wandbruntime‚2623  _runtime‚2623òôÃÔê&ÁäM PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 221449216 _runtime‚2623.376061444 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚68314425 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚30098002 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8603002 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚223879172811776 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚168 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚172 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚294280326117576 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚68304266 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3122398116584 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚30098000 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚294631436483072 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚294631453522432 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚294631436483072 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚68304266 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3130907209216 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚28796450 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚220655143173632 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 trainer/global_step‚3799 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚68314425 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚38206266 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚20193452 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚220655049063936 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚38206266 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚291157928000992 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚291157910961632 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚38216423 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚294631453522432 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚297762343692288 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚20193448 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 _step‚311 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚297762449398784 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 150006272 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3130995876352 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚38216423 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚30098002 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚30098000 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3224029638144 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚297762449398784 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚8603170FZ^#ÂéM UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚30098002 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚30098000 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3224029638144 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚297762449398784 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚8603170 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 221449216 _runtime‚2623.376061444 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚68314425 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚30098002 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8603002 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚223879172811776 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚168 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚172 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚294280326117576 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚68304266 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3122398116584 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚30098000 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚294631436483072 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚294631453522432 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚294631436483072 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚68304266 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3130907209216 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚28796450 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚220655143173632 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 trainer/global_step‚3799 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚68314425 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚38206266 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚20193452 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚220655049063936 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚38206266 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚291157928000992 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚291157910961632 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚38216423 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚294631453522432 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚297762343692288 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚20193448 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 _step‚311 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚297762449398784 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 150006272 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3130995876352 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚38216423 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0·a'ÿm2Ã- _wandbruntime‚2623  _runtime‚2623˜r„bÄý .loss/FRt1/detection_loss‚0.637482704160389 % _timestamp‚1.7844629489894738e+09 /loss/FRt1/assignment_loss‚2.625679531893953 2loss/SRqqt2/assignment_loss‚0.5916733166159963 /loss/FBt1/assignment_loss‚0.212184773651605 1loss/FBt2/assignment_loss‚0.31824721329669664 /loss/FRt2/detection_loss‚1.3556354503773238 1loss/SRqqt2/detection_loss‚0.7283722869882894 /loss/FBt2/detection_loss‚1.4098300369265158 %loss/total_loss‚11.35728982913675 0loss/FRt2/assignment_loss‚1.1739085077618558 1loss/SRqqt1/detection_loss‚1.0685177440827265 /loss/FBt1/detection_loss‚0.5644586517507071 _step‚312 epoch‚9 trainer/global_step‚3799 _runtime‚2623.377811597 2loss/SRqqt1/assignment_loss‚0.6712996116306907wžÅ‚ /loss/FBt2/detection_loss‚1.4098300369265158 %loss/total_loss‚11.35728982913675 0loss/FRt2/assignment_loss‚1.1739085077618558 1loss/SRqqt1/detection_loss‚1.0685177440827265 /loss/FBt1/detection_loss‚0.5644586517507071 _step‚312 epoch‚9 trainer/global_step‚3799 _runtime‚2623.377811597 2loss/SRqqt1/assignment_loss‚0.6712996116306907 .loss/FRt1/detection_loss‚0.637482704160389 % _timestamp‚1.7844629489894738e+09 /loss/FRt1/assignment_loss‚2.625679531893953 2loss/SRqqt2/assignment_loss‚0.5916733166159963 /loss/FBt1/assignment_loss‚0.212184773651605 1loss/FBt2/assignment_loss‚0.31824721329669664 /loss/FRt2/detection_loss‚1.3556354503773238 1loss/SRqqt2/detection_loss‚0.7283722869882894¸Å<²ŸÆ‚(:” æüòÒñíÕ¤ gpu.0.gpu‚67.0 gpu.0.memory‚16, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚47.0 gpu.0.powerWatts‚114.763' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚38.254333333333335 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0N C·jÇ‚(:ß æüòҨᤠdisk.nvme0n1.in‚ 1852.4453125% disk.nvme0n1.out‚30006.1748046875 disk.nvme1n1.in‚ 129.765625 disk.dm-0.out‚ 43671.2265625 disk.rbd0.in‚0 cpu‚25.85671455026835" disk.nvme0c0n1.in‚ 1852.4453125' disk.nvme0c0n1.out‚30006.1748046875' proc.memory.availableMB‚ 65.28515625* proc.memory.percent‚18.101048469543457 network.sent‚ 5.592117e+06 network.recv‚709927% disk./.usageGB‚1170.9811325073242# disk.nvme1n1.out‚15522.48828125 disk.md0.in‚0 disk.md0.out‚ 64.640625 disk.dm-0.in‚ 1982.2109375! disk.dm-1.out‚1718.2666015625) disk./.usagePercent‚76.27299348687994 disk.nvme1c1n1.in‚ 129.765625% disk.nvme1c1n1.out‚15522.48828125 disk.dm-1.in‚0 disk.rbd0.out‚0$ memory_percent‚99.60153102874756# proc.memory.rssMB‚ 2965.67578125 proc.cpu.threads‚17SA½ÿvÈ‚(:ë õüòÒƒˆ¿¤& disk.nvme1n1.out‚15523.40673828125 disk.md0.in‚0$ memory_percent‚99.59743022918701' proc.memory.availableMB‚ 65.95703125# proc.memory.rssMB‚ 2965.67578125 network.recv‚713006* proc.memory.percent‚18.101048469543457 network.sent‚ 5.65354e+06( disk./.usagePercent‚76.2730411938902! disk.dm-1.out‚1718.3759765625" disk.nvme0c0n1.in‚ 1852.4453125 disk.nvme0n1.in‚ 1852.4453125 disk.nvme1n1.in‚ 129.765625 cpu‚25.859037468898602 proc.cpu.threads‚17 disk.dm-0.in‚ 1982.2109375" disk.dm-0.out‚43672.5166015625 disk.dm-1.in‚0 disk.rbd0.in‚0 disk.rbd0.out‚0( disk.nvme0c0n1.out‚30007.29736328125& disk.nvme0n1.out‚30007.29736328125 disk.md0.out‚ 64.94140625% disk./.usageGB‚1170.9818649291992 disk.nvme1c1n1.in‚ 129.765625( disk.nvme1c1n1.out‚15523.40673828125›3̞ɂ(:“ õüòÒ½…¥ gpu.0.gpu‚90.0 gpu.0.memory‚16, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚48.0 gpu.0.powerWatts‚223.425' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚74.47500000000001 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ެʂ(:’ „ýòÒý춤 gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚47.0 gpu.0.powerWatts‚134.154' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚44.717999999999996 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0AòÇqË‚(:æ „ýòÒÜÊɤ disk.md0.out‚65.3125# proc.memory.rssMB‚ 2965.67578125" disk.nvme0c0n1.in‚ 1852.4453125 disk.nvme1n1.in‚ 129.765625 disk.rbd0.in‚0 disk.rbd0.out‚0$ memory_percent‚99.59897994995117 cpu‚25.861795594892843 proc.cpu.threads‚17% disk./.usageGB‚1170.9819107055664 disk.nvme0n1.in‚ 1852.4453125( disk.nvme1c1n1.out‚15524.08642578125 disk.dm-0.in‚ 1982.2109375 disk.dm-1.in‚0" disk.dm-1.out‚1718.48486328125% proc.memory.availableMB‚ 65.703125* proc.memory.percent‚18.101048469543457 network.sent‚ 5.660277e+06 network.recv‚716757) disk./.usagePercent‚76.27304417557835' disk.nvme0c0n1.out‚30008.2236328125% disk.nvme0n1.out‚30008.2236328125 disk.nvme1c1n1.in‚ 129.765625 disk.md0.in‚0" disk.dm-0.out‚43673.2001953125& disk.nvme1n1.out‚15524.08642578125[´T¹2Ì- _wandbruntime‚2657  _runtime‚2657¶¤£ÉÍà _runtime‚2657.767433764 _step‚313 ' lr-AdamW/pg1‚0.00012887214949400242 ' lr-AdamW/pg2‚0.00012887214949400242 trainer/global_step‚3849 % _timestamp‚1.7844629833790352e+09t÷oDÎÎÈ ' lr-AdamW/pg2‚0.00012887214949400242 trainer/global_step‚3849 % _timestamp‚1.7844629833790352e+09 _runtime‚2657.767433764 _step‚313 ' lr-AdamW/pg1‚0.00012887214949400242¹/ÃÍó2Ï- _wandbruntime‚2657  _runtime‚2657ù €4à'ÐÚO HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚29149681 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚30471821 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 150006272 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 127346688 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 _runtime‚2657.768696406 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚301483241565696 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚38689298 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3263765953024 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚8708489 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚226669657798656 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 221456384 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚301483347265024 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚298313770131968 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3263638606336 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3160853152680 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚298313770131968 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3169560093696 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚223405891845632 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚297958913072094 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚294798146705840 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚301483241565696 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚190 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3169471433728 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚69161119 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚298313787171328 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚20441192 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚30471821 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚29149871 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚38699441 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚20441196 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚186ÿÛ¢å'ÑßO YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚298313770131968 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3169560093696 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚223405891845632 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚297958913072094 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚294798146705840 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚301483241565696 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚190 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3169471433728 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚69161119 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚298313787171328 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚20441192 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚30471821 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚29149871 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚38699441 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚20441196 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚186 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚29149681 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚30471821 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 150006272 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 127346688 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 _runtime‚2657.768696406 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚301483241565696 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚38689298 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3263765953024 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚8708489 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚226669657798656 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 221456384 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚301483347265024 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚298313770131968 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3263638606336 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3160853152680 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960º`†ù2Ò- _wandbruntime‚2658  _runtime‚2658í;AÓäM XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 150006272 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 _runtime‚2658.192689748 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚8710813 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚176 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚38699118 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚69188727 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚30479452 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚38699118 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚29156885 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚69188727 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚298388936837632 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 trainer/global_step‚3849 _step‚315 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚20446252 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-;#©1 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚226726605269504 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚294872437231232 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3161636647392 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚298388919798272 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3170257149440 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚30479450 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚29157065 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚223461935463936 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 215157760 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚298388936837632 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚223462029573632 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3170257149440 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚298388919798272 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚69178568 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚20446248 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8710637 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚301559282654208 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3161549860854 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚30479450 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚29157065 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚223461935463936 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 215157760 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚298388936837632 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚223462029573632 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3170257149440 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚298388919798272 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚69178568 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚20446248 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8710637 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚301559282654208 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3161549860854 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚38699118 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚69188727 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚30479452 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚38699118 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚29156885 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚69188727 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚298388936837632 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 trainer/global_step‚3849 _step‚315 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚20446252 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360»›Gtã2Õ- _wandbruntime‚2658  _runtime‚2658/mãéÖ‚ 1loss/FBt1/assignment_loss‚0.11311342971643844 /loss/FRt2/detection_loss‚1.5092946921409092 1loss/SRqqt2/detection_loss‚0.7117279020474895 epoch‚9 trainer/global_step‚3849 0loss/FRt2/assignment_loss‚1.2248905167241013 1loss/FBt2/assignment_loss‚0.13524144721684245 %loss/total_loss‚11.37373920306262 % _timestamp‚1.7844629838052483e+09 _runtime‚2658.193390959 _step‚316 0loss/FRt1/assignment_loss‚2.5019468113828665 2loss/SRqqt2/assignment_loss‚0.6358820653966708 /loss/FRt1/detection_loss‚0.5653281740768276 1loss/SRqqt1/detection_loss‚1.1549674603903375 0loss/FBt1/detection_loss‚0.45961919584603955 /loss/FBt2/detection_loss‚1.5947674282922828 2loss/SRqqt1/assignment_loss‚0.7669600798318139°§“® ׇ 1loss/FBt1/assignment_loss‚0.11311342971643844 /loss/FRt2/detection_loss‚1.5092946921409092 1loss/SRqqt2/detection_loss‚0.7117279020474895 epoch‚9 trainer/global_step‚3849 0loss/FRt2/assignment_loss‚1.2248905167241013 1loss/FBt2/assignment_loss‚0.13524144721684245 %loss/total_loss‚11.37373920306262 % _timestamp‚1.7844629838052483e+09 _runtime‚2658.193390959 _step‚316 0loss/FRt1/assignment_loss‚2.5019468113828665 2loss/SRqqt2/assignment_loss‚0.6358820653966708 /loss/FRt1/detection_loss‚0.5653281740768276 1loss/SRqqt1/detection_loss‚1.1549674603903375 0loss/FBt1/detection_loss‚0.45961919584603955 /loss/FBt2/detection_loss‚1.5947674282922828 2loss/SRqqt1/assignment_loss‚0.7669600798318139¼àU©:yØ‚(:î “ýòÒÁÞã¤* proc.memory.percent‚18.101048469543457 disk.nvme1c1n1.in‚ 129.765625( disk.nvme1c1n1.out‚15524.44580078125 disk.md0.in‚0 disk.md0.out‚ 65.50390625" disk.dm-1.out‚1718.54345703125 disk.rbd0.out‚0 cpu‚25.864706779407427 network.sent‚ 5.663241e+06( disk./.usagePercent‚76.2730454179484 disk.nvme1n1.in‚ 129.765625 disk.rbd0.in‚0$ memory_percent‚99.59766864776611 disk.nvme0n1.in‚ 1852.4453125& disk.nvme0n1.out‚30010.03955078125& disk.nvme1n1.out‚15524.44580078125 disk.dm-0.in‚ 1982.2109375# proc.memory.rssMB‚ 2965.67578125 proc.cpu.threads‚17 network.recv‚719822% disk./.usageGB‚1170.9819297790527" disk.nvme0c0n1.in‚ 1852.4453125( disk.nvme0c0n1.out‚30010.03955078125# disk.dm-0.out‚43674.90283203125 disk.dm-1.in‚0' proc.memory.availableMB‚ 65.91796875Ž žÙ‚(:“ “ýòÒÚãÄ¥ gpu.0.gpu‚83.0 gpu.0.memory‚20, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚47.0 gpu.0.powerWatts‚120.802' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚40.26733333333334 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Ho(ŒqÚ‚(:æ ¢ýòÒèÌá¤# disk.nvme1n1.out‚15525.27734375# disk.dm-0.out‚43675.60205078125( disk.nvme0c0n1.out‚30011.27099609375 disk.nvme0n1.in‚ 1852.4453125& disk.nvme0n1.out‚30011.27099609375% disk.nvme1c1n1.out‚15525.27734375 disk.dm-1.in‚0$ memory_percent‚99.59766864776611 disk.nvme1n1.in‚ 129.765625 disk.md0.in‚0 disk.md0.out‚ 66.109375! disk.dm-1.out‚1718.6240234375* proc.memory.percent‚18.101048469543457 network.recv‚724127 disk.rbd0.in‚0 disk.dm-0.in‚ 1982.2109375' proc.memory.availableMB‚ 65.91796875# proc.memory.rssMB‚ 2965.67578125 cpu‚25.867219448487706 proc.cpu.threads‚17 network.sent‚ 5.727774e+06) disk./.usagePercent‚76.09606533053643% disk./.usageGB‚1168.2648429870605 disk.rbd0.out‚0" disk.nvme0c0n1.in‚ 1852.4453125 disk.nvme1c1n1.in‚ 129.765625ýjóÑÛ‚(:† ¢ýòÒü¥”¥ gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚47.0 gpu.0.powerWatts‚113.049' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚37.683 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0wµá2Ü- _wandbruntime‚2692  _runtime‚2692âªMFÉÝà _step‚317 ' lr-AdamW/pg1‚0.00012857726715805168 ' lr-AdamW/pg2‚0.00012857726715805168 trainer/global_step‚3899 % _timestamp‚1.7844630178369129e+09 _runtime‚2692.225305288§É^[ÎÞÈ ' lr-AdamW/pg1‚0.00012857726715805168 ' lr-AdamW/pg2‚0.00012857726715805168 trainer/global_step‚3899 % _timestamp‚1.7844630178369129e+09 _runtime‚2692.225305288 _step‚317½y,«2ß- _wandbruntime‚2692  _runtime‚2692Þ½ìà'àÚO M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚29509985 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚70035421 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3208821373952 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 125249536 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚298512690014800 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚302071253447168 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚301712781698288 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚30853271 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚30853273 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 150006272 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚39192293 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚229517089567232 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚185 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚226212778245632 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚305280180520448 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚20693992 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3200004897062 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚226212684135936 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚305280074821120 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚302071270486528 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚189 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚298512672975440 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚305280180520448 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚30853273 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚30853271 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3208910033920 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚229516870208000 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3208910033920 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3208821373952 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚29510174 ßP¨Ä áßO UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚305280074821120 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 219359232 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3200091683488 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 _runtime‚2692.226182959 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚305280074821120 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚302071270486528 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚189 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚298512672975440 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚305280180520448 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚30853273 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚30853271 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3208910033920 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚229516870208000 [KDeviceS`·õ¡!tatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3208910033920 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3208821373952 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚29510174 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚29509985 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚70035421 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3208821373952 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 125249536 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚298512690014800 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚302071253447168 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚301712781698288 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚30853271 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚30853273 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 150006272 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚39192293 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚229517089567232 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚185 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚226212778245632 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚305280180520448 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚20693992 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3200004897062 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚226212684135936 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 IDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚302146420152832 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚226268915973632 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚298586980540192 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 trainer/global_step‚3899 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚70063029 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚30860902 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚305356115909632 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3209695756800 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 213060608 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3200875178200 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚39202127 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 150006272 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚305356010203136 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚30860902 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚8818392 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 118950912 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3304998271488 _runtime‚2692.643790153 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8818225 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3209695756800 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚39202127 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚70052870 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚302146420152832 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚30860900 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3209607089664 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3200788391662 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚39191970 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚29517273 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚226268821863936 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚301787855718392 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚30860900 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚70063029 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0²ÔbOï&äéM TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 213060608 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3200875178200 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚39202127 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 150006272 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚305356010203136 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚30860902 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚8818392 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 118950912 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3304998271488 _runtime‚2692.643790153 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8818225 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3209695756800 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚39202127 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚70052870 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚302146420152832 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚30860900 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3209607089664 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3200788391662 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚39191970 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚29517273 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚226268821863936 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚301787855718392 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚30860900 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚70063029 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚302146420152832 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚226268915973632 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚298586980540192 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 trainer/global_step‚3899 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚70063029 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚30860902 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚305356115909632 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3209695756800¿¼.»½2å- _wandbruntime‚2692  _runtime‚2692–ïdæý /loss/FBt2/detection_loss‚0.7741958886384964 epoch‚9 _runtime‚2692.644538094 0loss/SRqqt1/assignment_loss‚0.68320264716167 2loss/SRqqt2/assignment_loss‚0.6098007901020374 /loss/FRt1/detection_loss‚0.5867955047781606 trainer/global_step‚3899 % _timestamp‚1.7844630182563257e+09 1loss/FBt1/assignment_loss‚0.05659907932226663 &loss/total_loss‚10.607078803652442 0loss/FRt2/assignment_loss‚1.1894028716861542 1loss/FBt2/assignment_loss‚0.21749745468088122 /loss/FBt1/detection_loss‚0.6495530949810515 _step‚320 /loss/FRt1/assignment_loss‚2.579728283098334 .loss/FRt2/detection_loss‚1.422572961585713 0loss/SRqqt1/detection_loss‚1.132090769732452 1loss/SRqqt2/detection_loss‚0.7056394578852233‡¬»°ç‚ /loss/FBt1/detection_loss‚0.6495530949810515 _step‚320 /loss/FRt1/assignment_loss‚2.579728283098334 .loss/FRt2/detection_loss‚1.422572961585713 0loss/SRqqt1/detection_loss‚1.132090769732452 1loss/SRqqt2/detection_loss‚0.7056394578852233 /loss/FBt2/detection_loss‚0.7741958886384964 epoch‚9 _runtime‚2692.644538094 0loss/SRqqt1/assignment_loss‚0.68320264716167 2loss/SRqqt2/assignment_loss‚0.6098007901020374 /loss/FRt1/detection_loss‚0.5867955047781606 trainer/global_step‚3899 % _timestamp‚1.7844630182563257e+09 1loss/FBt1/assignment_loss‚0.05659907932226663 &loss/total_loss‚10.607078803652442 0loss/FRt2/assignment_loss‚1.1894028716861542 1loss/FBt2/assignment_loss‚0.21749745468088122ÀXÁûkè‚(:à ±ýòң㋥) disk./.usagePercent‚76.09605936716015 disk.md0.out‚66.34375 disk.rbd0.in‚0 disk.nvme1c1n1.in‚ 129.765625' proc.memory.availableMB‚ 65.87890625* proc.memory.percent‚18.101048469543457% disk.nvme0c0n1.out‚30015.26171875 network.sent‚ 5.743641e+06 disk.dm-0.in‚ 1982.2109375! disk.dm-0.out‚43679.783203125! disk.dm-1.out‚1718.6943359375 disk.rbd0.out‚0" disk.nvme0c0n1.in‚ 1852.4453125# disk.nvme0n1.out‚30015.26171875 disk.md0.in‚0 disk.nvme0n1.in‚ 1852.4453125% disk.nvme1n1.out‚15526.0634765625# proc.memory.rssMB‚ 2965.67578125 cpu‚25.87006590753257 proc.cpu.threads‚17% disk./.usageGB‚1168.2647514343262 disk.dm-1.in‚0' disk.nvme1c1n1.out‚15526.0634765625 disk.nvme1n1.in‚ 129.765625$ memory_percent‚99.59790706634521 network.recv‚727357ˆÁÌ4Ÿé‚(:” ±ýòÒõþ¹¥ gpu.0.gpu‚95.0 gpu.0.memory‚23, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚47.0 gpu.0.powerWatts‚146.795' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚48.931666666666665 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0žµaìrê‚(:ç ÀýòÒ棂¥ disk.md0.in‚0 disk.rbd0.in‚0 disk.rbd0.out‚0" disk.nvme0c0n1.in‚ 1852.4453125 disk.nvme1c1n1.in‚ 129.765625* proc.memory.percent‚18.101048469543457 network.recv‚731372) disk./.usagePercent‚76.08057421827293" disk.dm-1.out‚1718.76708984375 disk.nvme0n1.in‚ 1852.4453125 disk.nvme1n1.in‚ 129.765625& disk.nvme1n1.out‚15526.59912109375& proc.memory.availableMB‚ 65.8828125# proc.memory.rssMB‚ 2965.67578125! disk.dm-0.out‚43680.412109375 disk.dm-1.in‚0( disk.nvme1c1n1.out‚15526.59912109375 cpu‚25.872430977158114 proc.cpu.threads‚17 network.sent‚ 5.795132e+06% disk./.usageGB‚1168.0270156860352 disk.md0.out‚ 66.69921875 disk.dm-0.in‚ 1982.2109375& disk.nvme0c0n1.out‚30016.189453125$ disk.nvme0n1.out‚30016.189453125# memory_percent‚99.5978832244873DL†`žë‚(:“ ÀýòҼа¥ gpu.0.gpu‚49.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚47.0 gpu.0.powerWatts‚113.469' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚37.82299999999999 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ˆc2ì- _wandbruntime‚2727  _runtime‚2727ï DÉíà % _timestamp‚1.7844630527594397e+09 _runtime‚2727.148030041 _step‚321 ' lr-AdamW/pg1‚0.00012827859819019715 ' lr-AdamW/pg2‚0.00012827859819019715 trainer/global_step‚3949¯~´³ÎîÈ trainer/global_step‚3949 % _timestamp‚1.7844630527594397e+09 _runtime‚2727.148030041 _step‚321 ' lr-AdamW/pg1‚0.00012827859819019715 ' lr-AdamW/pg2‚0.00012827859819019715Áüc)2ï- _wandbruntime‚2727  _runtime‚2727Š‡ÑÜðÚO TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3248171314176 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚309077013775872 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚29869158 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚302227199245040 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚39685145 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚39675002 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚194 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚190 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚232364243327488 trainer/global_step‚3949 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚31234721 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 129443840 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚309076908076544 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚305828753801728 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚70919868 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3239243427870 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚232364466881024 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3239330214296 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 223553536 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚229019664645632 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚31234723 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚8922362 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚70909723 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚39685145 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3248171314176 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚309076908076544 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚305828736762368 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚302227216284400 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚309077013775872 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚305828753801728 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚29868964 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3344672791552 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 _step‚322 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 % _timestamp‚1.7844630527608914e+09lADå'ñßO QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚29869158 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚302227199245040 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚39685145 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚39675002 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚194 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚190 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚232364243327488 trainer/global_step‚3949 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚31234721 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 129443840 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚309076908076544 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚305828753801728 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚70919868 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3239243427870 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚232364466881024 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3239330214296 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 223553536 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚229019664645632 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚31234723 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚8922362 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚70909723 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚39685145 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3248171314176 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚309076908076544 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚305828736762368 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚302227216284400 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚309077013775872 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚305828753801728 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚29868964 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3344672791552 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 _step‚322 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 % _timestamp‚1.7844630527608914e+09 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3248171314176 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚309077013775872 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200ÂK|Ï2ò- _wandbruntime‚2727  _runtime‚2727p¤Ô±é&óãM RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 $ _timestamp‚1.784463053171764e+09 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚31242350 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚31242352 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚229075802373632 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 _runtime‚2727.560357613 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚70937331 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚305903903468032 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚20951848 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3345477282304 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚302301489770432 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3240113709008 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚39694979 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 231934976 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 137825280 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3248957029888 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚70937331 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚305541620518800 _step‚323 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚39684822 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚309152843458560 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3240026922470 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚31242352 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚192 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚232421185546240 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚309152949165056 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚188 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚229075708263936 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚305541516692902 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚39684822 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3248957029888 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚31242350 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚39694979 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚29876131 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8924283 trainer/global_step‚3949 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚8924471Q˜3 ôèM E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3248957029888 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚70937331 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚305541620518800 _step‚323 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚39684822 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚309152843458560 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3240026922470 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚31242352 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚192 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚232421185546240 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 F@DeviceS°|UßtatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚309152949165056 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚188 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚229075708263936 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚305541516692902 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚39684822 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3248957029888 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚31242350 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚39694979 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚29876131 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚8924283 trainer/global_step‚3949 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚8924471 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 $ _timestamp‚1.784463053171764e+09 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚31242350 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚31242352 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚229075802373632 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 _runtime‚2727.560357613 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚70937331 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚305903903468032 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚20951848 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3345477282304 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚302301489770432 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3240113709008 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚39694979 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 231934976 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 137825280 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0ÃøŠÕ2õ- _wandbruntime‚2727  _runtime‚2727 ŒEçö‚ 1loss/SRqqt1/detection_loss‚1.1378805431027774 epoch‚9 1loss/FBt1/assignment_loss‚0.06136588270250058 1loss/FBt2/assignment_loss‚0.25585106771997607 /loss/FRt2/detection_loss‚1.3559266318068832 trainer/global_step‚3949 _runtime‚2727.560865994 0loss/FRt1/assignment_loss‚2.5510015818363336 1loss/SRqqt2/detection_loss‚0.7332851078969052 /loss/FBt1/detection_loss‚0.7076134727691457 /loss/FBt2/detection_loss‚1.3700538757774565 &loss/total_loss‚11.352696350451088 % _timestamp‚1.7844630531728275e+09 _step‚324 0loss/FRt2/assignment_loss‚1.2426856275734823 2loss/SRqqt1/assignment_loss‚0.6789962848100705 2loss/SRqqt2/assignment_loss‚0.6793348068725764 /loss/FRt1/detection_loss‚0.5787014675829809Ä™y ÷‡ 0loss/FRt2/assignment_loss‚1.2426856275734823 2loss/SRqqt1/assignment_loss‚0.6789962848100705 2loss/SRqqt2/assignment_loss‚0.6793348068725764 /loss/FRt1/detection_loss‚0.5787014675829809 1loss/SRqqt1/detection_loss‚1.1378805431027774 epoch‚9 1loss/FBt1/assignment_loss‚0.06136588270250058 1loss/FBt2/assignment_loss‚0.25585106771997607 /loss/FRt2/detection_loss‚1.3559266318068832 trainer/global_step‚3949 _runtime‚2727.560865994 0loss/FRt1/assignment_loss‚2.5510015818363336 1loss/SRqqt2/detection_loss‚0.7332851078969052 /loss/FBt1/detection_loss‚0.7076134727691457 /loss/FBt2/detection_loss‚1.3700538757774565 &loss/total_loss‚11.352696350451088 % _timestamp‚1.7844630531728275e+09 _step‚324Ä4¬nø‚(:ã ÏýòÒ†Ú¾¥# proc.memory.rssMB‚ 2965.67578125" disk.dm-1.out‚1719.22412109375 disk.rbd0.in‚0! disk.nvme0n1.out‚ 31927.734375 disk.md0.out‚ 67.00390625$ memory_percent‚99.59843158721924* proc.memory.percent‚18.101048469543457 cpu‚25.87503319899038 proc.cpu.threads‚17( disk.nvme1c1n1.out‚15563.33056640625 disk.nvme1n1.in‚ 129.765625" disk.dm-0.out‚45626.0654296875' proc.memory.availableMB‚ 65.79296875 network.recv‚734684# disk.nvme0c0n1.out‚ 31927.734375 disk.nvme0n1.in‚ 1852.4453125 disk.nvme1c1n1.in‚ 129.765625& disk.nvme1n1.out‚15563.33056640625 disk.dm-0.in‚ 1982.2109375 network.sent‚ 5.798054e+06) disk./.usagePercent‚76.27231341350974% disk./.usageGB‚1170.9706916809082 disk.dm-1.in‚0 disk.rbd0.out‚0" disk.nvme0c0n1.in‚ 1852.4453125 disk.md0.in‚0·; ù‚(:’ ÏýòÒçí ¥ gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚47.0 gpu.0.powerWatts‚174.813' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚58.270999999999994 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0,Ö›©jú‚(:ß ÞýòÒ‹š…¥ disk.md0.in‚0 disk.dm-0.in‚ 1982.2109375 disk.dm-1.in‚0 disk.rbd0.in‚0* proc.memory.percent‚18.101048469543457" disk.dm-1.out‚1719.59423828125 disk.rbd0.out‚0' disk.nvme1c1n1.out‚15564.1982421875# memory_percent‚99.6002197265625# proc.memory.rssMB‚ 2965.67578125 cpu‚25.876446243183857 network.recv‚739192) disk./.usagePercent‚76.32210934478556 disk.md0.out‚ 67.3671875( disk.nvme0c0n1.out‚32872.34619140625 disk.nvme0n1.in‚ 1852.4453125& disk.nvme0n1.out‚32872.34619140625 disk.nvme1c1n1.in‚ 129.765625 proc.memory.availableMB‚65.5% disk./.usageGB‚1171.7351837158203 disk.nvme1n1.in‚ 129.765625 disk.dm-0.out‚ 46571.890625" disk.nvme0c0n1.in‚ 1852.4453125 proc.cpu.threads‚17 network.sent‚ 5.862583e+06% disk.nvme1n1.out‚15564.1982421875Sž užû‚(:“ ÞýòÒ³¶¥ gpu.0.gpu‚91.0 gpu.0.memory‚19, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚49.0 gpu.0.powerWatts‚213.757' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚71.25233333333334 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0 ¾O#2ü- _wandbruntime‚2759  _runtime‚2759×jpÇýÁ & lr-AdamW/pg1‚0.0001279761617733885 & lr-AdamW/pg2‚0.0001279761617733885 trainer/global_step‚3999 % _timestamp‚1.7844630853814714e+09 _runtime‚2759.770564452 _step‚325%­T4ÌþÆ & lr-AdamW/pg1‚0.0001279761617733885 & lr-AdamW/pg2‚0.0001279761617733885 trainer/global_step‚3999 % _timestamp‚1.7844630853814714e+09 _runtime‚2759.770564452 _step‚325Å~ÉÖi2ÿ- _wandbruntime‚2759  _runtime‚2759é.÷¯à'€ÚO IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚312873741331968 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚309586237116928 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚31616173 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚231826456935936 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3287609914368 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 231942144 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3278568745104 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚31616171 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚312873847031296 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 trainer/global_step‚3999 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚9028441 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚309586237116928 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚31616171 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚30228220 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚9028624 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 _step‚326 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚309586220077568 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚235211589756416 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚309220207473318 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚40167854 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 150006272 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚305941742554000 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚40177997 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚231826551045632 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3385270652928 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚40177997 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚309220311299104 % _timestamp‚1.7844630853822265e+09 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3278481958678 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚309586220077568 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚312873741331968 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚309586237116928 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚31616173 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚231826456935936 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3287609914368 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 231942144 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3278568745104 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚31616171 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚312873847031296 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 trainer/global_step‚3999 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚9028441 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚309586237116928 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚31616171 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚30228220 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚9028624 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 _step‚326 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚309586220077568 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚235211589756416 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚309220207473318 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚40167854 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 150006272 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚305941742554000 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚40177997 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚231826551045632 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3385270652928 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚40177997 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚309220311299104 % _timestamp‚1.7844630853822265e+09 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3278481958678 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚309586220077568 IDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚9030572 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚309661369743872 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 150006272 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚309661386783232 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3279352239816 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚309661369743872 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚40177674 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚30235220 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚179 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚231882594663936 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3288395637248 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3288306970112 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚71801474 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3385945052672 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 _runtime‚2760.186279769 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3386074489344 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚306016033079392 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚31623800 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚71811633 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚71801474 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3279265453278 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 trainer/global_step‚3999 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚312949782420480 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0-µX°ï&†éM _runtime‚2760.186279769 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3386074489344 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚306016033079392 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚31623800 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚71811633 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚71801474 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3279265453278 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 trainer/global_step‚3999 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚312949782420480 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3288306970112 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚71811633 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚40187831 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚231882688773632 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚30235403 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚21204648 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚21204652 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 223546368 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚31623802 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚9030572 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚309661369743872 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 150006272 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚309661386783232 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3279352239816 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚309661369743872 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚40177674 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚30235220 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚179 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚231882594663936 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3288395637248 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3288306970112 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚71801474 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3385945052672 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812Ç=ÕY2‡- _wandbruntime‚2760  _runtime‚2760b'Cˆþ /loss/FBt2/detection_loss‚1.0688778634571159 $ _timestamp‚1.784463085798762e+09 /loss/FRt1/assignment_loss‚2.541367999517201 1loss/FBt2/assignment_loss‚0.07319145855646335 /loss/FBt1/detection_loss‚0.7871506873416888 epoch‚9 _step‚328 /loss/FRt1/detection_loss‚0.5602175214633064 &loss/total_loss‚10.962595547094576 trainer/global_step‚3999 0loss/FRt2/assignment_loss‚1.1565857631377254 2loss/SRqqt1/assignment_loss‚0.7980571595941567 .loss/FRt2/detection_loss‚1.379055774273266 1loss/SRqqt2/detection_loss‚0.6865723882901829 _runtime‚ 2760.18686816 2loss/SRqqt2/assignment_loss‚0.6758389899138516 1loss/FBt1/assignment_loss‚0.11264430662796077 1loss/SRqqt1/detection_loss‚1.1230356349216568U¿ÉU ‰ƒ 0loss/FRt2/assignment_loss‚1.1565857631377254 2loss/SRqqt1/assignment_loss‚0.7980571595941567 .loss/FRt2/detection_loss‚1.379055774273266 1loss/SRqqt2/detection_loss‚0.6865723882901829 _runtime‚ 2760.18686816 2loss/SRqqt2/assignment_loss‚0.6758389899138516 1loss/FBt1/assignment_loss‚0.11264430662796077 1loss/SRqqt1/detection_loss‚1.1230356349216568 /loss/FBt2/detection_loss‚1.0688778634571159 $ _timestamp‚1.784463085798762e+09 /loss/FRt1/assignment_loss‚2.541367999517201 1loss/FBt2/assignment_loss‚0.07319145855646335 /loss/FBt1/detection_loss‚0.7871506873416888 epoch‚9 _step‚328 /loss/FRt1/detection_loss‚0.5602175214633064 &loss/total_loss‚10.962595547094576 trainer/global_step‚3999È#—2Š- _wandbruntime‚2773  _runtime‚2773’ϨÝ(‹É{ 8 Purity/2FRt1SRqqt1FBt/FRt_purity‚0.41898149251937866 particle/accuracy_4_of_5‚1 ?&Purity/2FRt1SRqqt0FBt/event_proportion‚0.011928013525903225 7 Purity/*FRt*SRqqt2FBt/FRt_purity‚0.8565004467964172 =&Purity/*FRt1SRqqt1FBt/event_proportion‚0.0188569575548172 9"Purity/*FRt2SRqqt0FBt/SRqqt_purity‚0.6873858571052551 8"Purity/0FRt1SRqqt0FBt/SRqqt_purity‚0.655156672000885 ("Purity/1FRt2SRqqt1FBt/event_purity‚0 8 Purity/*FRt*SRqqt*FBt/FRt_purity‚0.17188337445259094 9"Purity/*FRt*SRqqt2FBt/event_purity‚0.8433111310005188 6 Purity/*FRt1SRqqt2FBt/FBt_purity‚0.898148238658905 9"Purity/0FRt0SRqqt2FBt/event_purity‚0.9714980125427246 & Purity/1FRt2SRqqt*FBt/FBt_purity‚1 ("Purity/2FRt2SRqqt0FBt/event_purity‚0 *jet/accuracy_1_of_3‚0.9213441014289856 7 Purity/1FRt1SRqqt2FBt/FBt_purity‚0.8888888955116272 & Purity/2FRt*SRqqt*FBt/FBt_purity‚1 ,"Purity/2FRt*SRqqt2FBt/event_purity‚0.375 & Purity/2FRt1SRqqt1FBt/FBt_purity‚1 8 Purity/*FRt*SRqqt0FBt/FRt_purity‚0.15868936479091644 :"Purity/*FRt0SRqqt*FBt/event_purity‚0.24743933975696564 7 Purity/0FRt*SRqqt1FBt/FBt_purity‚0.9969416856765747 ?&Purity/2FRt2SRqqt*FBt/event_proportion‚0.006417410913854837 8"Purity/0FRt*SRqqt*FBt/SRqqt_purity‚0.643526017665863 , Purity/0FRt2SRqqt1FBt/FBt_purity‚0.96875 @&Purity/1FRt1SRqqt1FBt/event_proportion‚0.0064639137126505375 jet/accuracy_4_of_5‚0 7 Purity/1FRt0SRqqt2FBt/FRt_purity‚0.8801019787788391 7 Purity/1FRt1SRqqt*FBt/FBt_purity‚0.9931973218917847 & Purity/2FRt*SRqqt2FBt/FBt_purity‚1 trainer/global_step‚3999 9"Purity/*FRt*SRqqt1FBt/event_purity‚0.7240235209465027 7 Purity/*FRt1SRqqt*FBt/FBt_purity‚0.9862610697746277 =&Purity/*FRt1SRqqt0FBt/event_proportion‚0.3670479953289032 <&Purity/*FRt2SRqqt*FBt/event_proportion‚0.054222472012043 =&Purity/0FRt*SRqqt*FBt/event_proportion‚0.2839471697807312 ?&Purity/0FRt0SRqqt2FBt/event_proportion‚0.007626487873494625 9"Purity/1FRt*SRqqt0FBt/SRqqt_purity‚0.7473211884498596 >&Purity/1FRt0SRqqt1FBt/event_proportion‚0.00969587080180645 ("Purity/*FRt2SRqqt1FBt/event_purity‚0 7 Purity/0FRt1SRqqt1FBt/FBt_purity‚0.9901987910270691 ;"Purity/1FRt1SRqqt0FBt/event_purity‚0.031237689778208733 :"Purity/2FRt0SRqqt0FBt/event_purity‚0.03139110282063484 ("Purity/1FRt2SRqqt0FBt/event_purity‚0 *jet/accuracy_1_of_2‚0.7895927429199219 ("Purity/0FRt1SRqqt2FBt/event_purity‚0 _step‚329 ?&Purity/*FRt*SRqqt2FBt/event_proportion‚0.010997953824698925 7 Purity/1FRt*SRqqt1FBt/FRt_purity‚0.5342257618904114 =&Purity/1FRt1SRqqt*FBt/event_proportion‚0.1534598171710968 *jet/accuracy_1_of_4‚0.9629591703414917 7 Purity/*FRt0SRqqt2FBt/FBt_purity‚0.9881680607795715 =&Purity/*FRt1SRqqt*FBt/event_proportion‚0.3866024911403656 8"Purity/*FRt1SRqqt0FBt/event_purity‚0.384609580039978 9"Purity/0FRt*SRqqt0FBt/event_purity‚0.6283787488937378 ?&Purity/2FRt2SRqqt0FBt/event_proportion‚0.006417410913854837 5 Purity/1FRt0SRqqt0FBt/FRt_purity‚0.18123859167099 ,&Purity/1FRt2SRqqt2FBt/event_proportion‚0 ;"Purity/2FRt*SRqqt0FBt/event_purity‚0.015876781195402145 8 Purity/2FRt2SRqqt*FBt/FRt_purity‚0.06915096193552017 9"Purity/2FRt2SRqqt*FBt/SRqqt_purity‚0.8069889545440674 jet/accuracy_3_of_5‚1 9"Purity/0FRt*SRqqt1FBt/SRqqt_purity‚0.5516076683998108 /"Purity/2FRt0SRqqt1FBt/event_purity‚0.234375 9"Purity/2FRt1SRqqt*FBt/SRqqt_purity‚0.7506262063980103 9"Purity/1FRt*SRqqt1FBt/SRqqt_purity‚0.6578471064567566 7 Purity/1FRt*SRqqt1FBt/FBt_purity‚0.9982993602752686 8 Purity/1FRt0SRqqt*FBt/FRt_purity‚0.19417908787727356 8 Purity/2FRt2SRqqt0FBt/FRt_purity‚0.06915096193552017 =&Purity/0FRt*SRqqt0FBt/event_proportion‚0.2322126179933548 8 Purity/1FRt*SRqqt0FBt/FRt_purity‚0.15912434458732605 9"Purity/1FRt2SRqqt*FBt/SRqqt_purity‚0.7215824127197266 9"Purity/2FRt*SRqqt1FBt/event_purity‚0.1133333295583725 *jet/accuracy_2_of_3‚0.5234445929527283 particle/accuracy_2_of_4‚1 9"Purity/*FRt1SRqqt1FBt/SRqqt_purity‚0.6121011972427368 7 Purity/*FRt2SRqqt*FBt/FBt_purity‚0.9814814329147339 =&Purity/0FRt0SRqqt*FBt/event_proportion‚0.0390159972012043 >&Purity/1FRt*SRqqt1FBt/event_proportion‚0.01643880270421505 @&Purity/1FRt0SRqqt2FBt/event_proportion‚0.0024879092816263437 *jet/accuracy_1_of_1‚0.3172316551208496 & Purity/2FRt0SRqqt2FBt/FBt_purity‚1 particle/accuracy_2_of_5‚1 <&Purity/*FRt*SRqqt0FBt/event_proportion‚0.881045401096344 =&Purity/*FRt*SRqqt1FBt/event_proportion‚0.0617094486951828 8 Purity/1FRt*SRqqt*FBt/FRt_purity‚0.17203891277313232 7 Purity/1FRt0SRqqt1FBt/FBt_purity‚0.9968254566192627 9"Purity/2FRt1SRqqt1FBt/SRqqt_purity‚0.7314814329147339 =&Purity/*FRt0SRqqt*FBt/event_proportion‚0.5129278302192688 ("Purity/*FRt1SRqqt2FBt/event_purity‚0 ("Purity/0FRt2SRqqt1FBt/event_purity‚0 6 Purity/1FRt*SRqqt2FBt/FRt_purity‚0.866893470287323 9"Purity/1FRt1SRqqt*FBt/SRqqt_purity‚0.7498324513435364 @&Purity/2FRt*SRqqt1FBt/event_proportion‚0.0016043527284637094 9"Purity/0FRt1SRqqt*FBt/SRqqt_purity‚0.6497382521629333 9"Purity/1FRt1SRqqt1FBt/SRqqt_purity‚0.6803480982780457 particle/accuracy_5_of_5‚0 8"Purity/0FRt*SRqqt2FBt/event_purity‚0.914840579032898 =&Purity/0FRt1SRqqt*FBt/event_proportion‚0.2203543484210968 =&Purity/*FRt0SRqqt0FBt/event_proportion‚0.4607747495174408 9"Purity/0FRt1SRqqt1FBt/event_purity‚0.5169343948364258 @&Purity/0FRt2SRqqt1FBt/event_proportion‚0.0007207961170934141 9"Purity/1FRt0SRqqt*FBt/event_purity‚0.1926942616701126 =&Purity/1FRt0SRqqt0FBt/event_proportion‚0.4408714771270752 7 Purity/1FRt0SRqqt1FBt/FRt_purity‚0.6090885996818542 8"Purity/*FRt*SRqqt*FBt/SRqqt_purity‚0.690274715423584 7 Purity/*FRt*SRqqt1FBt/FBt_purity‚0.9974440336227417 @&Purity/*FRt1SRqqt2FBt/event_proportion‚0.0006975446594879031 ("Purity/1FRt1SRqqt2FBt/event_purity‚0 ;"Purity/2FRt*SRqqt*FBt/event_purity‚0.021923378109931946 8 Purity/2FRt1SRqqt0FBt/FRt_purity‚0.18614992499351501 particle/accuracy_1_of_2‚1 /particle/accuracy_4_of_4‚0.9611361622810364 :"Purity/*FRt*SRqqt0FBt/event_purity‚0.26214152574539185 @&Purity/1FRt1SRqqt2FBt/event_proportion‚0.0002325148816453293 +jet/accuracy_2_of_2‚0.13892701268196106 % _timestamp‚1.7844630993526938e+09 9"Purity/0FRt2SRqqt0FBt/SRqqt_purity‚0.6245522499084473 9"Purity/1FRt*SRqqt2FBt/SRqqt_purity‚0.2222222238779068 7 Purity/2FRt1SRqqt*FBt/FRt_purity‚0.2008141726255417 ("Purity/2FRt1SRqqt2FBt/event_purity‚0 8"Purity/*FRt1SRqqt*FBt/SRqqt_purity‚0.692802369594574 :"Purity/1FRt*SRqqt*FBt/event_purity‚0.14850836992263794 'particle/f_score‚0.6381506323814392 7 Purity/*FRt*SRqqt1FBt/FRt_purity‚0.5240254998207092 *jet/accuracy_2_of_4‚0.7695395946502686 /particle/accuracy_3_of_4‚0.9942177534103394 ("Purity/2FRt1SRqqt0FBt/event_purity‚0 ( Purity/2FRt1SRqqt2FBt/FRt_purity‚0.5 & Purity/0FRt0SRqqt1FBt/FBt_purity‚1 9"Purity/0FRt2SRqqt*FBt/SRqqt_purity‚0.6182807683944702 :"Purity/*FRt0SRqqt0FBt/event_purity‚0.17471371591091156 =&Purity/*FRt0SRqqt1FBt/event_proportion‚0.0418526791036129 >&Purity/2FRt0SRqqt0FBt/event_proportion‚0.01990327425301075 & Purity/2FRt0SRqqt*FBt/FBt_purity‚1 8"Purity/*FRt0SRqqt1FBt/event_purity‚0.886688232421875 9"Purity/*FRt1SRqqt1FBt/event_purity‚0.4048749804496765 :"Purity/*FRt2SRqqt0FBt/event_purity‚0.17716054618358612 8"Purity/0FRt1SRqqt0FBt/event_purity‚0.655156672000885 & Purity/1FRt2SRqqt1FBt/FRt_purity‚0 ,&Purity/2FRt2SRqqt1FBt/event_proportion‚0 jet/accuracy_2_of_5‚1 _runtime‚2773.741580572 :"Purity/*FRt*SRqqt*FBt/event_purity‚0.29886120557785034 =&Purity/*FRt*SRqqt*FBt/event_proportion‚0.9537528157234192 7 Purity/*FRt0SRqqt1FBt/FBt_purity‚0.9994588494300842 >&Purity/0FRt2SRqqt*FBt/event_proportion‚0.02457682229578495 9"Purity/2FRt*SRqqt*FBt/SRqqt_purity‚0.7734477519989014 =&Purity/2FRt*SRqqt0FBt/event_proportion‚0.0382486991584301 7 Purity/2FRt*SRqqt1FBt/FRt_purity‚0.4675000309944153 8 Purity/2FRt0SRqqt0FBt/FRt_purity‚0.16394305229187012 9"Purity/*FRt1SRqqt*FBt/event_purity‚0.3848888874053955 =&Purity/1FRt*SRqqt*FBt/event_proportion‚0.6297432780265808 9"Purity/1FRt*SRqqt*FBt/SRqqt_purity‚0.7430546879768372 7 Purity/1FRt*SRqqt2FBt/FBt_purity‚0.9888888597488403 ("Purity/2FRt1SRqqt1FBt/event_purity‚0 :"Purity/0FRt*SRqqt2FBt/SRqqt_purity‚0.19230769574642181 @&Purity/2FRt1SRqqt2FBt/event_proportion‚2.325148852833081e-05 :"Purity/*FRt1SRqqt2FBt/SRqqt_purity‚0.20370370149612427 <&Purity/2FRt*SRqqt*FBt/event_proportion‚0.040062315762043 9"Purity/2FRt*SRqqt1FBt/SRqqt_purity‚0.7314814329147339 8 Purity/2FRt0SRqqt*FBt/FRt_purity‚0.18355785310268402 jet/accuracy_1_of_5‚1 8&Purity/1FRt2SRqqt0FBt/event_proportion‚ 0.02294921875 & Purity/1FRt2SRqqt1FBt/FBt_purity‚1 @&Purity/2FRt*SRqqt2FBt/event_proportion‚0.0002092633949359879 +jet/accuracy_3_of_3‚0.06369408220052719 7 Purity/2FRt0SRqqt2FBt/FRt_purity‚0.7142857313156128 7 Purity/*FRt0SRqqt1FBt/FRt_purity‚0.5975667238235474 9"Purity/0FRt1SRqqt1FBt/SRqqt_purity‚0.5696097612380981 & Purity/2FRt*SRqqt1FBt/FBt_purity‚1 >&Purity/0FRt2SRqqt0FBt/event_proportion‚0.02385602705180645 7 Purity/0FRt*SRqqt*FBt/FBt_purity‚0.9922121167182922 ("Purity/0FRt0SRqqt1FBt/event_purity‚1 0"Purity/0FRt2SRqqt1FBt/SRqqt_purity‚ 0.4296875 A&Purity/1FRt2SRqqt1FBt/event_proportion‚0.00027901786961592734 & Purity/2FRt1SRqqt*FBt/FBt_purity‚1 :"Purity/0FRt2SRqqt*FBt/event_purity‚0.38865455985069275 , Purity/0FRt2SRqqt*FBt/FBt_purity‚0.96875 9"Purity/1FRt1SRqqt1FBt/event_purity‚0.2586972415447235 =&Purity/2FRt1SRqqt*FBt/event_proportion‚0.0127883180975914 9"Purity/2FRt2SRqqt0FBt/SRqqt_purity‚0.8069889545440674 particle/accuracy_1_of_3‚1 7 Purity/0FRt1SRqqt2FBt/FBt_purity‚0.9038461446762085 & Purity/1FRt1SRqqt1FBt/FBt_purity‚1 ("Purity/1FRt2SRqqt*FBt/event_purity‚0 A&Purity/2FRt0SRqqt2FBt/event_proportion‚0.00018601190822664648 7 Purity/*FRt1SRqqt1FBt/FBt_purity‚0.9937577247619629 =&Purity/1FRt1SRqqt0FBt/event_proportion‚0.1467633992433548 9"Purity/1FRt1SRqqt0FBt/SRqqt_purity‚0.7540692090988159 7 Purity/0FRt1SRqqt*FBt/FBt_purity‚0.9826012849807739 :"Purity/1FRt*SRqqt0FBt/event_purity‚0.13832175731658936 9"Purity/1FRt2SRqqt0FBt/SRqqt_purity‚0.7268150448799133 7 Purity/0FRt*SRqqt2FBt/FBt_purity‚0.9ݘò799469709396362 & Purity/2FRt1SRqqt2FBt/FBt_purity‚1 =&Purity/*FRt0SRqqt2FBt/event_proportion‚0.0103004090487957 8 Purity/*FRt2SRqqt0FBt/FRt_purity‚0.05703243613243103 9"Purity/0FRt1SRqqt*FBt/event_purity‚0.6464859843254089 6 Purity/1FRt1SRqqt1FBt/FRt_purity‚0.440080463886261 =&Purity/2FRt0SRqqt*FBt/event_proportion‚0.0208565853536129 ,&Purity/2FRt2SRqqt2FBt/event_proportion‚0 6 Purity/*FRt1SRqqt1FBt/FRt_purity‚0.443162739276886 9"Purity/1FRt*SRqqt2FBt/event_purity‚0.6823129653930664 ;"Purity/1FRt1SRqqt*FBt/event_purity‚0.040826715528964996 +jet/accuracy_3_of_4‚0.07413794100284576 ("Purity/2FRt1SRqqt2FBt/SRqqt_purity‚0 7 Purity/*FRt2SRqqt*FBt/FRt_purity‚0.0566127710044384 9"Purity/2FRt*SRqqt0FBt/SRqqt_purity‚0.7763167023658752 particle/accuracy_2_of_3‚1 9"Purity/*FRt1SRqqt0FBt/SRqqt_purity‚0.6979016661643982 9"Purity/0FRt*SRqqt0FBt/SRqqt_purity‚0.6489580869674683 7 Purity/1FRt*SRqqt*FBt/FBt_purity‚0.9952738881111145 9"Purity/1FRt1SRqqt2FBt/SRqqt_purity‚0.2222222238779068 jet/accuracy_5_of_5‚0 9"Purity/*FRt0SRqqt2FBt/event_purity‚0.9016159772872925 ("Purity/0FRt0SRqqt0FBt/event_purity‚1 9"Purity/1FRt0SRqqt2FBt/event_purity‚0.7495465278625488 ,&Purity/*FRt2SRqqt2FBt/event_proportion‚0 :"Purity/1FRt2SRqqt1FBt/SRqqt_purity‚0.32499998807907104 ) Purity/*FRt1SRqqt2FBt/FRt_purity‚0.75 7 Purity/*FRt2SRqqt1FBt/FBt_purity‚0.9814814329147339 ,&Purity/0FRt2SRqqt2FBt/event_proportion‚0 ("Purity/2FRt*SRqqt2FBt/SRqqt_purity‚0 :"Purity/*FRt*SRqqt2FBt/SRqqt_purity‚0.20370370149612427 7 Purity/0FRt0SRqqt2FBt/FBt_purity‚0.9857490062713623 7 Purity/1FRt0SRqqt*FBt/FBt_purity‚0.9965709447860718 7 Purity/1FRt1SRqqt2FBt/FRt_purity‚0.7777777910232544 particle/accuracy_1_of_1‚1 =&Purity/0FRt1SRqqt0FBt/event_proportion‚0.2083565890789032 7 Purity/1FRt1SRqqt*FBt/FRt_purity‚0.1255813091993332 >&Purity/1FRt2SRqqt*FBt/event_proportion‚0.02322823740541935 ("Purity/2FRt2SRqqt*FBt/event_purity‚0 /particle/accuracy_3_of_3‚0.9994933605194092 9"Purity/2FRt0SRqqt2FBt/event_purity‚0.4285714328289032 7 Purity/*FRt*SRqqt2FBt/FBt_purity‚0.9817334413528442 8 Purity/*FRt1SRqqt*FBt/FRt_purity‚0.13651332259178162 =&Purity/0FRt0SRqqt1FBt/event_proportion‚0.0313895083963871 & Purity/*FRt2SRqqt1FBt/FRt_purity‚0 ?&Purity/0FRt1SRqqt1FBt/event_proportion‚0.011555989272892475 7 Purity/1FRt1SRqqt0FBt/FRt_purity‚0.1100330799818039 @&Purity/2FRt0SRqqt1FBt/event_proportion‚0.0007672990905120969 ("Purity/2FRt1SRqqt*FBt/event_purity‚0 ,jet/accuracy_4_of_4‚0.013899499550461769 8 Purity/*FRt1SRqqt0FBt/FRt_purity‚0.12084418535232544 =&Purity/0FRt0SRqqt0FBt/event_proportion‚0.0462472103536129 6 Purity/*FRt*SRqqt*FBt/FBt_purity‚0.993362307548523 9"Purity/*FRt2SRqqt1FBt/SRqqt_purity‚0.3865740895271301 ?&Purity/0FRt*SRqqt2FBt/event_proportion‚0.008068266324698925 8 Purity/1FRt2SRqqt*FBt/FRt_purity‚0.04784741997718811 particle/accuracy_1_of_4‚1 :"Purity/*FRt2SRqqt*FBt/event_purity‚0.17384985089302063 :"Purity/0FRt2SRqqt0FBt/event_purity‚0.40017950534820557 8 Purity/2FRt*SRqqt*FBt/FRt_purity‚0.17096172273159027 particle/accuracy_1_of_5‚1 )loss/val_total_loss‚10.51936782731366 9"Purity/*FRt*SRqqt0FBt/SRqqt_purity‚0.6954104900360107 7 Purity/*FRt0SRqqt2FBt/FRt_purity‚0.8718065619468689 8&Purity/*FRt2SRqqt0FBt/event_proportion‚ 0.05322265625 =&Purity/0FRt*SRqqt1FBt/event_proportion‚0.0436662957072258 9 Purity/1FRt2SRqqt0FBt/FRt_purity‚0.048439569771289825 @&Purity/2FRt1SRqqt1FBt/event_proportion‚0.0008370535797439516 (particle/accuracy‚0.8484002351760864 8"Purity/*FRt*SRqqt1FBt/SRqqt_purity‚0.590260922908783 8 Purity/2FRt*SRqqt0FBt/FRt_purity‚0.15511992573738098 6validation_average_jet_accuracy‚0.3609776496887207 7 Purity/2FRt0SRqqt1FBt/FRt_purity‚0.5520833730697632 @&Purity/*FRt2SRqqt1FBt/event_proportion‚0.0009998140158131719 8"Purity/0FRt*SRqqt1FBt/event_purity‚0.853403627872467 :"Purity/2FRt0SRqqt*FBt/event_purity‚0.04317931830883026 +particle/sensitivity‚0.6225080490112305 8 Purity/*FRt0SRqqt*FBt/FRt_purity‚0.19315975904464722 :"Purity/1FRt*SRqqt1FBt/event_purity‚0.44092056155204773 /particle/accuracy_2_of_2‚0.9998962879180908 *particle/specificity‚0.910180926322937 :"Purity/0FRt1SRqqt2FBt/SRqqt_purity‚0.19230769574642181 epoch‚9 8"Purity/0FRt*SRqqt*FBt/event_purity‚0.671514630317688 7 Purity/0FRt0SRqqt*FBt/FBt_purity‚0.9954611659049988 @&Purity/0FRt1SRqqt2FBt/event_proportion‚0.0004417782765813172 7"Purity/1FRt0SRqqt0FBt/event_purity‚0.18123859167099 9"Purity/1FRt0SRqqt1FBt/event_purity‚0.5765407085418701 7 Purity/1FRt0SRqqt2FBt/FBt_purity‚0.9970238208770752 9"Purity/2FRt1SRqqt0FBt/SRqqt_purity‚0.7547144889831543 + Purity/2FRt*SRqqt2FBt/FRt_purity‚0.6875 8 Purity/*FRt0SRqqt0FBt/FRt_purity‚0.17974671721458435 & Purity/2FRt0SRqqt1FBt/FBt_purity‚1 9"Purity/*FRt2SRqqt*FBt/SRqqt_purity‚0.6818234920501709 =&Purity/1FRt*SRqqt0FBt/event_proportion‚0.6105840802192688 @&Purity/1FRt*SRqqt2FBt/event_proportion‚0.0027204242069274187 particle/accuracy_3_of_5‚1 7 Purity/*FRt0SRqqt*FBt/FBt_purity‚0.9958749413490295 9"Purity/0FRt0SRqqt*FBt/event_purity‚0.9945608973503113 =&Purity/1FRt0SRqqt*FBt/event_proportion‚0.4530552327632904­úqÔ=ŒÎ{ >&Purity/0FRt2SRqqt0FBt/event_proportion‚0.02385602705180645 7 Purity/0FRt*SRqqt*FBt/FBt_purity‚0.9922121167182922 ("Purity/0FRt0SRqqt1FBt/event_purity‚1 0"Purity/0FRt2SRqqt1FBt/SRqqt_purity‚ 0.4296875 A&Purity/1FRt2SRqqt1FBt/event_proportion‚0.00027901786961592734 & Purity/2FRt1SRqqt*FBt/FBt_purity‚1 :"Purity/0FRt2SRqqt*FBt/event_purity‚0.38865455985069275 , Purity/0FRt2SRqqt*FBt/FBt_purity‚0.96875 9"Purity/1FRt1SRqqt1FBt/event_purity‚0.2586972415447235 =&Purity/2FRt1SRqqt*FBt/event_proportion‚0.0127883180975914 9"Purity/2FRt2SRqqt0FBt/SRqqt_purity‚0.8069889545440674 particle/accuracy_1_of_3‚1 7 Purity/0FRt1SRqqt2FBt/FBt_purity‚0.9038461446762085 & Purity/1FRt1SRqqt1FBt/FBt_purity‚1 ("Purity/1FRt2SRqqt*FBt/event_purity‚0 A&Purity/2FRt0SRqqt2FBt/event_proportion‚0.00018601190822664648 7 Purity/*FRt1SRqqt1FBt/FBt_purity‚0.9937577247619629 =&Purity/1FRt1SRqqt0FBt/event_proportion‚0.1467633992433548 9"Purity/1FRt1SRqqt0FBt/SRqqt_purity‚0.7540692090988159 7 Purity/0FRt1SRqqt*FBt/FBt_purity‚0.9826012849807739 :"Purity/1FRt*SRqqt0FBt/event_purity‚0.13832175731658936 9"Purity/1FRt2SRqqt0FBt/SRqqt_purity‚0.7268150448799133 7 Purity/0FRt*SRqqt2FBt/FBt_purity‚0.9799469709396362 & Purity/2FRt1SRqqt2FBt/FBt_purity‚1 =&Purity/*FRt0SRqqt2FBt/event_proportion‚0.0103004090487957 8 Purity/*FRt2SRqqt0FBt/FRt_purity‚0.05703243613243103 9"Purity/0FRt1SRqqt*FBt/event_purity‚0.6464859843254089 6 Purity/1FRt1SRqqt1FBt/FRt_purity‚0.440080463886261 =&Purity/2FRt0SRqqt*FBt/event_proportion‚0.0208565853536129 ,&Purity/2FRt2SRqqt2FBt/event_proportion‚0 6 Purity/*FRt1SRqqt1FBt/FRt_purity‚0.443162739276886 9"Purity/1FRt*SRqqt2FBt/event_purity‚0.6823129653930664 ;"Purity/1FRt1SRqqt*FBt/event_purity‚0.040826715528964996 +jet/accuracy_3_of_4‚0.07413794100284576 ("Purity/2FRt1SRqqt2FBt/SRqqt_purity‚0 7 Purity/*FRt2SRqqt*FBt/FRt_purity‚0.0566127710044384 9"Purity/2FRt*SRqqt0FBt/SRqqt_purity‚0.7763167023658752 particle/accuracy_2_of_3‚1 9"Purity/*FRt1SRqqt0FBt/SRqqt_purity‚0.6979016661643982 9"Purity/0FRt*SRqqt0FBt/SRqqt_purity‚0.6489580869674683 7 Purity/1FRt*SRqqt*FBt/FBt_purity‚0.9952738881111145 9"Purity/1FRt1SRqqt2FBt/SRqqt_purity‚0.2222222238779068 jet/accuracy_5_of_5‚0 9"Purity/*FRt0SRqqt2FBt/event_purity‚0.9016159772872925 ("Purity/0FRt0SRqqt0FBt/event_purity‚1 9"Purity/1FRt0SRqqt2FBt/event_purity‚0.7495465278625488 ,&Purity/*FRt2SRqqt2FBt/event_proportion‚0 :"Purity/1FRt2SRqqt1FBt/SRqqt_purity‚0.32499998807907104 ) Purity/*FRt1SRqqt2FBt/FRt_purity‚0.75 7 Purity/*FRt2SRqqt1FBt/FBt_purity‚0.9814814329147339 ,&Purity/0FRt2SRqqt2FBt/event_proportion‚0 ("Purity/2FRt*SRqqt2FBt/SRqqt_purity‚0 :"Purity/*FRt*SRqqt2FBt/SRqqt_purity‚0.20370370149612427 7 Purity/0FRt0SRqqt2FBt/FBt_purity‚0.9857490062713623 7 Purity/1FRt0SRqqt*FBt/FBt_purity‚0.9965709447860718 7 Purity/1FRt1SRqqt2FBt/FRt_purity‚0.7777777910232544 particle/accuracy_1_of_1‚1 =&Purity/0FRt1SRqqt0FBt/event_proportion‚0.2083565890789032 7 Purity/1FRt1SRqqt*FBt/FRt_purity‚0.1255813091993332 >&Purity/1FRt2SRqqt*FBt/event_proportion‚0.02322823740541935 ("Purity/2FRt2SRqqt*FBt/event_purity‚0 /particle/accuracy_3_of_3‚0.9994933605194092 9"Purity/2FRt0SRqqt2FBt/event_purity‚0.4285714328289032 7 Purity/*FRt*SRqqt2FBt/FBt_purity‚0.9817334413528442 8 Purity/*FRt1SRqqt*FBt/FRt_purity‚0.13651332259178162 =&Purity/0FRt0SRqqt1FBt/event_proportion‚0.0313895083963871 & Purity/*FRt2SRqqt1FBt/FRt_purity‚0 ?&Purity/0FRt1SRqqt1FBt/event_proportion‚0.011555989272892475 7 Purity/1FRt1SRqqt0FBt/FRt_purity‚0.1100330799818039 @&Purity/2FRt0SRqqt1FBt/event_proportion‚0.0007672990905120969 ("Purity/2FRt1SRqqt*FBt/event_purity‚0 ,jet/accuracy_4_of_4‚0.013899499550461769 8 Purity/*FRt1SRqqt0FBt/FRt_purity‚0.12084418535232544 =&Purity/0FRt0SRqqt0FBt/event_proportion‚0.0462472103536129 6 Purity/*FRt*SRqqt*FBt/FBt_purity‚0.993362307548523 9"Purity/*FRt2SRqqt1FBt/SRqqt_purity‚0.3865740895271301 ?&Purity/0FRt*SRqqt2FBt/event_proportion‚0.008068266324698925 8 Purity/1FRt2SRqqt*FBt/FRt_purity‚0.04784741997718811 particle/accuracy_1_of_4‚1 :"Purity/*FRt2SRqqt*FBt/event_purity‚0.17384985089302063 :"Purity/0FRt2SRqqt0FBt/event_purity‚0.40017950534820557 8 Purity/2FRt*SRqqt*FBt/FRt_purity‚0.17096172273159027 particle/accuracy_1_of_5‚1 )loss/val_total_loss‚10.51936782731366 9"Purity/*FRt*SRqqt0FBt/SRqqt_purity‚0.6954104900360107 7 Purity/*FRt0SRqqt2FBt/FRt_purity‚0.8718065619468689 8&Purity/*FRt2SRqqt0FBt/event_proportion‚ 0.05322265625 =&Purity/0FRt*SRqqt1FBt/event_proportion‚0.0436662957072258 9 Purity/1FRt2SRqqt0FBt/FRt_purity‚0.048439569771289825 @&Purity/2FRt1SRqqt1FBt/event_proportion‚0.0008370535797439516 (particle/accuracy‚0.8484002351760864 8"Purity/*FRt*SRqqt1FBt/SRqqt_purity‚0.590260922908783 8 Purity/2FRt*SRqqt0FBt/FRt_purity‚0.15511992573738098 6validation_average_jet_accuracy‚0.3609776496887207 7 Purity/2FRt0SRqqt1FBt/FRt_purity‚0.5520833730697632 @&Purity/*FRt2SRqqt1FBt/event_proportion‚0.0009998140158131719 8"Purity/0FRt*SRqqt1FBt/event_purity‚0.853403627872467 :"Purity/2FRt0SRqqt*FBt/event_purity‚0.04317931830883026 +particle/sensitivity‚0.6225080490112305 8 Purity/*FRt0SRqqt*FBt/FRt_purity‚0.19315975904464722 :"Purity/1FRt*SRqqt1FBt/event_purity‚0.44092056155204773 /particle/accuracy_2_of_2‚0.9998962879180908 *particle/specificity‚0.910180926322937 :"Purity/0FRt1SRqqt2FBt/SRqqt_purity‚0.19230769574642181 epoch‚9 8"Purity/0FRt*SRqqt*FBt/event_purity‚0.671514630317688 7 Purity/0FRt0SRqqt*FBt/FBt_purity‚0.9954611659049988 @&Purity/0FRt1SRqqt2FBt/event_proportion‚0.0004417782765813172 7"Purity/1FRt0SRqqt0FBt/event_purity‚0.18123859167099 9"Purity/1FRt0SRqqt1FBt/event_purity‚0.5765407085418701 7 Purity/1FRt0SRqqt2FBt/FBt_purity‚0.9970238208770752 9"Purity/2FRt1SRqqt0FBt/SRqqt_purity‚0.7547144889831543 + Purity/2FRt*SRqqt2FBt/FRt_purity‚0.6875 8 Purity/*FRt0SRqqt0FBt/FRt_purity‚0.17974671721458435 & Purity/2FRt0SRqqt1FBt/FBt_purity‚1 9"Purity/*FRt2SRqqt*FBt/SRqqt_purity‚0.6818234920501709 =&Purity/1FRt*SRqqt0FBt/event_proportion‚0.6105840802192688 @&Purity/1FRt*SRqqt2FBt/event_proportion‚0.0027204242069274187 particle/accuracy_3_of_5‚1 7 Purity/*FRt0SRqqt*FBt/FBt_purity‚0.9958749413490295 9"Purity/0FRt0SRqqt*FBt/event_purity‚0.9945608973503113 =&Purity/1FRt0SRqqt*FBt/event_proportion‚0.4530552327632904 8 Purity/2FRt1SRqqt1FBt/FRt_purity‚0.41898149251937866 particle/accuracy_4_of_5‚1 ?&Purity/2FRt1SRqqt0FBt/event_proportion‚0.011928013525903225 7 Purity/*FRt*SRqqt2FBt/FRt_purity‚0.8565004467964172 =&Purity/*FRt1SRqqt1FBt/event_proportion‚0.0188569575548172 9"Purity/*FRt2SRqqt0FBt/SRqqt_purity‚0.6873858571052551 8"Purity/0FRt1SRqqt0FBt/SRqqt_purity‚0.655156672000885 ("Purity/1FRt2SRqqt1FBt/event_purity‚0 8 Purity/*FRt*SRqqt*FBt/FRt_purity‚0.17188337445259094 9"Purity/*FRt*SRqqt2FBt/event_purity‚0.8433111310005188 6 Purity/*FRt1SRqqt2FBt/FBt_purity‚0.898148238658905 9"Purity/0FRt0SRqqt2FBt/event_purity‚0.9714980125427246 & Purity/1FRt2SRqqt*FBt/FBt_purity‚1 ("Purity/2FRt2SRqqt0FBt/event_purity‚0 *jet/accuracy_1_of_3‚0.9213441014289856 7 Purity/1FRt1SRqqt2FBt/FBt_purity‚0.8888888955116272 & Purity/2FRt*SRqqt*FBt/FBt_purity‚1 ,"Purity/2FRt*SRqqt2FBt/event_purity‚0.375 & Purity/2FRt1SRqqt1FBt/FBt_purity‚1 8 Purity/*FRt*SRqqt0FBt/FRt_purity‚0.15868936479091644 :"Purity/*FRt0SRqqt*FBt/event_purity‚0.24743933975696564 7 Purity/0FRt*SRqqt1FBt/FBt_purity‚0.9969416856765747 ?&Purity/2FRt2SRqqt*FBt/event_proportion‚0.006417410913854837 8"Purity/0FRt*SRqqt*FBt/SRqqt_purity‚0.643526017665863 , Purity/0FRt2SRqqt1FBt/FBt_purity‚0.96875 @&Purity/1FRt1SRqqt1FBt/event_proportion‚0.0064639137126505375 jet/accuracy_4_of_5‚0 7 Purity/1FRt0SRqqt2FBt/FRt_purity‚0.8801019787788391 7 Purity/1FRt1SRqqt*FBt/FBt_purity‚0.9931973218917847 & Purity/2FRt*SRqqt2FBt/FBt_purity‚1 trainer/global_step‚3999 9"Purity/*FRt*SRqqt1FBt/event_purity‚0.7240235209465027 7 Purity/*FRt1SRqqt*FBt/FBt_purity‚0.9862610697746277 =&Purity/*FRt1SRqqt0FBt/event_proportion‚0.3670479953289032 <&Purity/*FRt2SRqqt*FBt/event_proportion‚0.054222472012043 =&Purity/0FRt*SRqqt*FBt/event_proportion‚0.2839471697807312 ?&Purity/0FRt0SRqqt2FBt/event_proportion‚0.007626487873494625 9"Purity/1FRt*SRqqt0FBt/SRqqt_purity‚0.7473211884498596 >&Purity/1FRt0SRqqt1FBt/event_proportion‚0.00969587080180645 ("Purity/*FRt2SRqqt1FBt/event_purity‚0 7 Purity/0FRt1SRqqt1FBt/FBt_purity‚0.9901987910270691 ;"Purity/1FRt1SRqqt0FBt/event_purity‚0.031237689778208733 :"Purity/2FRt0SRqqt0FBt/event_purity‚0.03139110282063484 ("Purity/1FRt2SRqqt0FBt/event_purity‚0 *jet/accuracy_1_of_2‚0.7895927429199219 ("Purity/0FRt1SRqqt2FBt/event_purity‚0 _step‚329 ?&Purity/*FRt*SRqqt2FBt/event_proportion‚0.010997953824698925 7 Purity/1FRt*SRqqt1FBt/FRt_purity‚0.5342257618904114 =&Purity/1FRt1SRqqt*FBt/event_proportion‚0.1534598171710968 *jet/accuracy_1_of_4‚0.9629591703414917 7 Purity/*FRt0SRqqt2FBt/FBt_purity‚0.9881680607795715 =&Purity/*FRt1SRqqt*FBt/event_proportion‚0.3866024911403656 8"Purity/*FRt1SRqqt0FBt/event_purity‚0.384609580039978 9"Purity/0FRt*SRqqt0FBt/event_purity‚0.6283787488937378 ?&Purity/2FRt2SRqqt0FBt/event_proportion‚0.006417410913854837 5 Purity/1FRt0SRqqt0FBt/FRt_purity‚0.18123859167099 ,&Purity/1FRt2SRqqt2FBt/event_proportion‚0 ;"Purity/2FRt*SRqqt0FBt/event_purity‚0.015876781195402145 8 Purity/2FRt2SRqqt*FBt/FRt_purity‚0.06915096193552017 9"Purity/2FRt2SRqqt*FBt/SRqqt_purity‚0.8069889545440674 jet/accuracy_3_of_5‚1 9"Purity/0FRt*SRqqt1FBt/SRqqt_purity‚0.5516076683998108 /"Purity/2FRt0SRqqt1FBt/event_purity‚0.234375 9"Purity/2FRt1SRqqt*FBt/SRqqt_purity‚0.7506262063980103 9"Purity/1FRt*SRqqt1FBt/SRqqt_purity‚0.6578471064567566 7 Purity/1FRt*SRqqt1FBt/FBt_purity‚0.9982993602752686 8 Purity/1FRt0SRqqt*FBt/FRt_purity‚0.19417908787727356 8 Purity/2FRt2SRqqt0FBt/FRt_purity‚0.06915096193552017 =&Purity/0FRt*SRqqt0FBt/event_proportion‚0.2322126179933548 8 Purity/1FRt*SRqqt0FBt/FRt_purity‚0.15912434458732605 9"Purity/1FRt2SRqqt*FBt/SRqqt_purity‚0.7215824127197266 9"Purity/2FRt*SRqqt1FBt/event_purity‚0.1133333295583725 *jet/accuracy_2_of_3‚0.5234445929527283 particle/accuracy_2_of_4‚1 9"Purity/*FRt1SRqqt1FBt/SRqqt_purity‚0.6121011972427368 7 Purity/*FRt2SRqqt*FBt/FBt_purity‚0.9814814329147339 =&Purity/0FRt0SRqqt*FBt/event_proportion‚0.0390159972012043 >&Purity/1FRt*SRqqt1FBt/event_proportion‚0.01643880270421505 @&Purity/1FRt0SRqqt2FBt/event_proportion‚0.0024879092816263437 *jet/accuracy_1_of_1‚0.3172316551208496 & Purity/2FRt0SRqqt2FBt/FBt_purity‚1 particle/accuracy_2_of_5‚1 <&Purity/*FRt*SRqqt0FBt/event_proportion‚0.881045401096344 =&Purity/*FRt*SRqqt1FBt/event_proportion‚0.0617094486951828 8 Purity/1FRt*SRqqt*FBt/FRt_purity‚0.17203891277313232 7 Purity/1FRt0SRqqt1FBt/FBt_purity‚0.9968254566192627 9"Purity/2FRt1SRqqt1FBt/SRqqt_purity‚0.7314814329147339 =&Purity/*FRt0SRqqt*FBt/event_proportion‚0.5129278302192688 ("Purity/*FRt1SRqqt2FBt/event_purity‚0 ("Purity/0FRt2SRqqt1FBt/event_purity‚0 6 Purity/1FRt*SRqqt2FBt/FRt_purity‚0.866893470287323 9"Purity/1FRt1SRqqt*FBt/SRqqt_purity‚0.7498324513435364 @&Purity/2FRt*SRqqt1FBt/event_proportion‚0.0016043527284637094 9"Purity/0FRt1SRqqt*FBt/SRqqt_purity‚0.6497382521629333 9"Purity/1FRt1SRqqt1FBt/SRqqt_purity‚0.6803480982780457 particle/accuracy_5_of_5‚0 8"Purity/0FRt*SRqqt2FBt/event_purity‚0.914840579032898 =&Purity/0FRt1SRqqt*FBt/event_proportion‚0.2203543484210968 =&Purity/*FRt0SRqqt0FBt/event_proportion‚0.4607747495174408 9"Purity/0FRt1SRqqt1FBt/event_purity‚0.5169343948364258 @&Purity/0FRt2SRqqt1FBt/event_proportion‚0.0007207961170934141 9"Purity/1FRt0SRqqt*FBt/event_purity‚0.1926942616701126 =&Purity/1FRt0SRqqt0FBt/event_proportion‚0.4408714771270752 7 Purity/1FRt0SRqqt1FBt/FRt_purity‚0.6090885996818542 8"Purity/*FRt*SRqqt*FBt/SRqqt_purity‚0.690274715423584 7 Purity/*FRt*SRqqt1FBt/FBt_purity‚0.9974440336227417 @&Purity/*FRt1SRqqt2FBt/event_proportion‚0.0006975446594879031 ("Purity/1FRt1SRqqt2FBt/event_purity‚0 ;"Purity/2FRt*SRqqt*FBt/event_purity‚0.021923378109931946 8 Purity/2FRt1SRqqt0FBt/FRt_purity‚0.18614992499351501 particle/accuracy_1_of_2‚1 /particle/accuracy_4_of_4‚0.9611361622810364 :"Purity/*FRt*SRqqt0FBt/event_purity‚0.26214152574539185 @&Purity/1FRt1SRqqt2FBt/event_proportion‚0.0002325148816453293 +jet/accuracy_2_of_2‚0.13892701268196106 % _timestamp‚1.7844630993526938e+09 9"Purity/0FRt2SRqqt0FBt/SRqqt_purity‚0.6245522499084473 9"Purity/1FRt*SRqqt2FBt/SRqqt_purity‚0.2222222238779068 7 Purity/2FRt1SRqqt*FBt/FRt_purity‚0.2008141726255417 ("Purity/2FRt1SRqqt2FBt/event_purity‚0 8"Purity/*FRt1SRqqt*FBt/SRqqt_purity‚0.692802369594574 :"Purity/1FRt*SRqqt*FBt/event_purity‚0.14850836992263794 'particle/f_score‚0.6381506323814392 7 Purity/*FRt*SRqqt1FBt/FRt_purity‚0.5240254998207092 *jet/accuracy_2_of_4‚0.7695395946502686 /particle/accuracy_3_of_4‚0.9942177534103394 ("Purity/2FRt1SRqqt0FBt/event_purity‚0 ( Purity/2FRt1SRqqt2FBt/FRt_purity‚0.5 & Purity/0FRt0SRqqt1FBt/FBt_purity‚1 9"Purity/0FRt2SRqqt*FBt/SRqqt_purity‚0.6182807683944702 :"Purity/*FRt0SRqqt0FBt/event_purity‚0.17471371591091156 =&Purity/*FRt0SRqqt1FBt/event_proportion‚0.0418526791036129 >&Purity/2FRt0SRqqt0FBt/event_proportion‚0.01990327425301075 & Purity/2FRt0SRqqt*FBt/FBt_purity‚1 8"Purity/*FRt0SRqqt1FBt/event_purity‚0.886688232421875 9"Purity/*FRt1SRqqt1FBt/event_purity‚0.4048749804496765 :"Purity/*FRt2SRqqt0FBt/event_purity‚0.17716054618358612 8"Purity/0FRt1SRqqt0FBt/event_purity‚0.655156672000885 & Purity/1FRt2SRqqt1FBt/FRt_purity‚0 ,&Purity/2FRt2SRqqt1FBt/event_proportion‚0 jet/accuracy_2_of_5‚1 _runtime‚2773.741580572 :"Purity/*FRt*SRqqt*FBt/event_purity‚0.29886120557785034 =&Purity/*FRt*SRqqt*FBt/event_proportion‚0.9537528157234192 7 Purity/*FRt0SRqqt1FBt/FBt_purity‚0.9994588494300842 >&Purity/0FRt2SRqqt*FBt/event_proportion‚0.02457682229578495 9"Purity/2FRt*SRqqt*FBt/SRqqt_purity‚0.7734477519989014 =&Purity/2FRt*SRqqt0FBt/event_proportion‚0.0382486991584301 7 Purity/2FRt*SRqqt1FBt/FRt_purity‚0.4675000309944153 8 Purity/2FRt0SRqqt0FBt/FRt_purity‚0.16394305229187012 9"Purity/*FRt1SRqqt*FBt/event_purity‚0.3848888874053955 =&Purity/1FRt*SRqqt*FBt/event_proportion‚0.6297432780265808 9"Purity/1FRt*SRqqt*FBt/SRqqt_purity‚0.7430546879768372 7 Purity/1FRt*SRqqt2FBt/FBt_purity‚0.9888888597488403 ("Purity/2FRt1SRqqt1FBt/event_purity‚0 :"Purity/0FRt*SRqqt2FBt/SRqqt_purity‚0.19230769574642181 @&Purity/2FRt1SRqqt2FBt/event_proportion‚2.325148852833081e-05 :"Purity/*FRt1SRqqt2FBt/SRqqt_purity‚0.20370370149612427 <&Purity/2FRt*SRqqt*FBt/event_proportion‚0.040062315762043 9"Purity/2FRt*SRqqt1FBt/SRqqt_purity‚0.7314814329147339 8 Purity/2FRt0SRqqt*FBt/FRt_purity‚0.18355785310268402 jet/accuracy_1_of_5‚1 8&Purity/1FRt2SRqqt0FBt/event_proportion‚ 0.02294921875 & Purity/1FRt2SRqqt1FBt/FBt_purity‚1 @&Purity/2FRt*SRqqt2FBt/event_proportion‚0.0002092633949359879 +jet/accuracy_3_of_3‚0.06369408220052719 7 Purity/2FRt0SRqqt2FBt/FRt_purity‚0.7142857313156128 7 Purity/*FRt0SRqqt1FBt/FRt_purity‚0.5975667238235474 9"Purity/0FRt1SRqqt1FBt/SRqqt_purity‚0.5696097612380981 & Purity/2FRt*SRqqt1FBt/FBt_purity‚1Ɉ­Ïj‚(:ß üýòÒЉ·¤ disk.md0.in‚0 disk.dm-1.in‚0! disk.nvme1c1n1.out‚ 15781.6875 disk.nvme1n1.in‚ 129.765625$ memory_percent‚99.19884204864502( proc.memory.availableMB‚ 131.26171875* proc.memory.percent‚18.101072311401367 network.sent‚ 5.917382e+06 network.recv‚746400 disk.dm-0.in‚ 1982.2109375 disk.rbd0.out‚0 disk.nvme0n1.in‚ 1852.4453125 disk.nvme1c1n1.in‚ 129.765625 disk.nvme1n1.out‚ 15781.6875 cpu‚25.88845637457471) disk./.usagePercent‚76.33466498507406 disk.md0.out‚ 68.08984375" disk.dm-0.out‚47700.2080078125" disk.nvme0c0n1.in‚ 1852.4453125( disk.nvme0c0n1.out‚33848.83447265625& disk.nvme0n1.out‚33848.83447265625" disk.dm-1.out‚1783.68603515625 disk.rbd0.in‚0" proc.memory.rssMB‚ 2965.6796875 proc.cpu.threads‚17% disk./.usageGB‚1171.9279441833496ê85›Ž‚(: üýòÒ–Áî¤ gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚47.0 gpu.0.powerWatts‚219.11' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚73.03666666666668 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0R±Œ v‚(:ë ‹þòÒž‰·¤ disk.rbd0.out‚0" proc.memory.rssMB‚ 2965.6796875 network.sent‚ 5.986177e+06 network.recv‚750569) disk./.usagePercent‚76.33466697286616% disk./.usageGB‚1171.9279747009277 disk.nvme1c1n1.in‚ 129.765625 disk.nvme1n1.in‚ 129.765625& disk.nvme1n1.out‚15820.59814453125 disk.md0.in‚0 disk.md0.out‚ 68.37890625 disk.rbd0.in‚0 disk.nvme0n1.in‚ 1852.4453125' proc.memory.availableMB‚ 131.3203125* proc.memory.percent‚18.101072311401367 proc.cpu.threads‚17( disk.nvme1c1n1.out‚15820.59814453125" disk.nvme0c0n1.in‚ 1852.4453125' disk.nvme0c0n1.out‚33859.1318359375% disk.nvme0n1.out‚33859.1318359375$ memory_percent‚99.19848442077637 cpu‚25.886912993123058 disk.dm-0.in‚ 1982.2109375" disk.dm-0.out‚47712.6806640625 disk.dm-1.in‚0" disk.dm-1.out‚1819.80322265625¥s¿“‚(:ˆ ‹þòÒâ©»¤ gpu.0.gpu‚73.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚48.0 gpu.0.powerWatts‚158.715' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚52.905 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ÝE®p‘‚(:å šþòÒ»î¤ disk.rbd0.in‚0 disk.nvme0n1.in‚ 1852.4453125% disk.nvme1n1.out‚15849.2216796875" disk.dm-0.out‚47713.2587890625" disk.nvme0c0n1.in‚ 1852.4453125( proc.memory.availableMB‚ 131.30859375 cpu‚25.885377650416785 network.recv‚754037) disk./.usagePercent‚76.33466697286616% disk./.usageGB‚1171.9279747009277 disk.rbd0.out‚0& disk.nvme0c0n1.out‚33860.138671875$ disk.nvme0n1.out‚33860.138671875" proc.memory.rssMB‚ 2965.6796875 proc.cpu.threads‚17 disk.nvme1c1n1.in‚ 129.765625 disk.nvme1n1.in‚ 129.765625 disk.md0.out‚ 68.90234375 disk.dm-1.out‚1847.740234375# memory_percent‚99.1985559463501* proc.memory.percent‚18.101072311401367 network.sent‚ 5.991797e+06' disk.nvme1c1n1.out‚15849.2216796875 disk.md0.in‚0 disk.dm-0.in‚ 1982.2109375 disk.dm-1.in‚0<Žœž’‚(:“ šþòÒîŠã¤ gpu.0.gpu‚90.0 gpu.0.memory‚18, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚47.0 gpu.0.powerWatts‚217.607' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚72.53566666666667 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0z£‹2“- _wandbruntime‚2806  _runtime‚2806’ÕiËɔà ' lr-AdamW/pg1‚0.00012766997733255163 ' lr-AdamW/pg2‚0.00012766997733255163 trainer/global_step‚4049 % _timestamp‚1.7844631324926565e+09 _runtime‚2806.881025677 _step‚330í¡›ÃÎ•È ' lr-AdamW/pg1‚0.00012766997733255163 ' lr-AdamW/pg2‚0.00012766997733255163 trainer/global_step‚4049 % _timestamp‚1.7844631324926565e+09 _runtime‚2806.881025677 _step‚330ÊŽãT2–- _wandbruntime‚2806  _runtime‚2806(Gå³È—ÚO ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 150006272 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚310629718820432 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3333222534478 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚21560651 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚235436708540928 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 trainer/global_step‚4049 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚317682491260416 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3342300571648 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚72872779 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚40753356 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 217262080 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚40743213 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚310629735859792 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚9183226 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 123152384 % _timestamp‚1.7844631324936478e+09 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚238878236214272 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚238878018952192 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚21560647 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚9183401 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚175 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚72862634 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚314340190688768 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚314340190688768 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3342389231616 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚32119423 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3441310411264 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 _step‚331 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚32119421 KBDeviceStatsMonitor.on_train_batch_start/ðZUÕ allocation.large_pool.peak‚1509 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3441433563648 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3333135748052 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3342300571648 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚235436802650624 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 _runtime‚2806.882009899 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3342389231616 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚317682596959744 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚314340207728128 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚317682491260416 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚3143402077281282®ÿ5å'˜ßO D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚235436802650624 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 _runtime‚2806.882009899 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3342389231616 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚317682596959744 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚314340207728128 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚317682491260416 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚314340207728128 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 150006272 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚310629718820432 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3333222534478 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚21560651 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚235436708540928 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 trainer/global_step‚4049 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚317682491260416 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3342300571648 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚72872779 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚40753356 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 217262080 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚40743213 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚310629735859792 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚9183226 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 123152384 % _timestamp‚1.7844631324936478e+09 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚238878236214272 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚238878018952192 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚21560647 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚9183401 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚175 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚72862634 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚314340190688768 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚314340190688768 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3342389231616 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚32119423 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3441310411264 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 _step‚331 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚32119421 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3441433563648 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3333135748052 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3342300571648 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130Ë»Dåý2™- _wandbruntime‚2807  _runtime‚2807׶'„ê&šäM UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚32127050 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3333919242652 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚176 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚310704026385184 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚32127052 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3343086287360 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚317758532348928 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚21565707 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚72890242 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚40753033 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚30751302 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 % _timestamp‚1.7844631332306967e+09 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚72890242 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚314415357394432 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚72880083 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚314415340355072 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3442120448000 _runtime‚2807.619228513 _step‚332 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚314415357394432 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚238935177680384 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚40763190 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚238934966716928 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚9185419 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3442237301760 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚317758426642432 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 210963456 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚314037928588476 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚30751122 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚314038032414374 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812<§>Ä$›éM C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚32127050 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3333919242652 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚176 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚310704026385184 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚32127052 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3343086287360 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚317758532348928 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 PDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚21565707 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚72890242 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚40753033 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚30751302 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 % _timestamp‚1.7844631332306967e+09 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚72890242 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚314415357394432 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚72880083 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚314415340355072 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3442120448000 _runtime‚2807.619228513 _step‚332 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚314415357394432 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚238935177680384 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚40763190 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚238934966716928 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚9185419 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3442237301760 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚317758426642432 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 210963456 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚314037928588476 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocateÏ Ñ+d‚130 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚30751122 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚314038032414374 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0Ì'ÝN"2œ- _wandbruntime‚2807  _runtime‚2807‘id‹ÿ 0loss/FRt2/assignment_loss‚1.1968893904145985 1loss/SRqqt2/detection_loss‚0.6570138231303254 2loss/SRqqt1/assignment_loss‚0.6151489495416675 1loss/FBt1/assignment_loss‚0.10178298990699246 trainer/global_step‚4049 % _timestamp‚1.7844631332317157e+09 _step‚333 epoch‚10 2loss/SRqqt2/assignment_loss‚0.6554677150500087 1loss/FBt2/assignment_loss‚0.30202597117742525 /loss/FRt2/detection_loss‚1.4887811151250925 0loss/SRqqt1/detection_loss‚1.044227154062682 /loss/FBt1/detection_loss‚0.6079169743593553 /loss/FBt2/detection_loss‚1.2176249858377308 _runtime‚2807.619804304 .loss/FRt1/assignment_loss‚2.52380236511022 /loss/FRt1/detection_loss‚0.5833240362523725 %loss/total_loss‚10.99400546996847¼•ŸÝ ž„ trainer/global_step‚4049 % _timestamp‚1.7844631332317157e+09 _step‚333 epoch‚10 2loss/SRqqt2/assignment_loss‚0.6554677150500087 1loss/FBt2/assignment_loss‚0.30202597117742525 /loss/FRt2/detection_loss‚1.4887811151250925 0loss/SRqqt1/detection_loss‚1.044227154062682 /loss/FBt1/detection_loss‚0.6079169743593553 /loss/FBt2/detection_loss‚1.2176249858377308 _runtime‚2807.619804304 .loss/FRt1/assignment_loss‚2.52380236511022 /loss/FRt1/detection_loss‚0.5833240362523725 %loss/total_loss‚10.99400546996847 0loss/FRt2/assignment_loss‚1.1968893904145985 1loss/SRqqt2/detection_loss‚0.6570138231303254 2loss/SRqqt1/assignment_loss‚0.6151489495416675 1loss/FBt1/assignment_loss‚0.10178298990699246Í~,¨sŸ‚(:è ©þòÒŸ”Ѥ% disk./.usageGB‚1171.9278259277344 disk.md0.out‚ 69.31640625" disk.dm-0.out‚47716.9345703125( disk.nvme1c1n1.out‚15850.46435546875 proc.memory.rssMB‚ 2965.6875 cpu‚25.883783117555982 network.recv‚757107 disk.rbd0.out‚0 disk.nvme0n1.in‚ 1852.4453125 disk.nvme1c1n1.in‚ 129.765625 network.sent‚ 5.994761e+06 disk.nvme1n1.in‚ 129.765625 disk.md0.in‚0 disk.dm-1.out‚1847.849609375 disk.rbd0.in‚0$ memory_percent‚99.19819831848145' proc.memory.availableMB‚ 131.3671875 proc.cpu.threads‚17) disk./.usagePercent‚76.33465728237971& disk.nvme1n1.out‚15850.46435546875 disk.dm-0.in‚ 1982.2109375 disk.dm-1.in‚0" disk.nvme0c0n1.in‚ 1852.4453125( disk.nvme0c0n1.out‚33863.56494140625& disk.nvme0n1.out‚33863.56494140625* proc.memory.percent‚18.101119995117188yZù¡ ‚(:’ ©þòÒÞô³¥ gpu.0.gpu‚40.0 gpu.0.memory‚6, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚48.0 gpu.0.powerWatts‚220.831' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚73.61033333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ÓúÈp¡‚(:å ¸þòÒçÓ¤* proc.memory.percent‚18.101119995117188) disk./.usagePercent‚76.33465976711982 disk.nvme1n1.in‚ 129.765625% disk.nvme1n1.out‚15852.9931640625 disk.md0.out‚ 69.7421875 disk.dm-0.in‚ 1982.2109375 disk.rbd0.out‚0 proc.cpu.threads‚17 network.sent‚ 6.058141e+06 disk.nvme0n1.in‚ 1852.4453125 disk.dm-1.in‚0 disk.rbd0.in‚0 proc.memory.rssMB‚ 2965.6875 cpu‚25.882200636606356 network.recv‚761153$ disk./.usageGB‚1171.927864074707& disk.nvme0n1.out‚33864.83056640625" disk.dm-0.out‚47717.9599609375" disk.dm-1.out‚1849.70849609375 disk.nvme1c1n1.in‚ 129.765625' disk.nvme1c1n1.out‚15852.9931640625 disk.md0.in‚0" disk.nvme0c0n1.in‚ 1852.4453125( disk.nvme0c0n1.out‚33864.83056640625# memory_percent‚99.1997241973877' proc.memory.availableMB‚ 131.1171875ïöé‘¢‚(:† ¸þòÒå×›¥ gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚47.0 gpu.0.powerWatts‚186.045' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚62.015 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0‹áÇâ2£- _wandbruntime‚2840  _runtime‚2840[m,Ȥ ' lr-AdamW/pg1‚0.00012736006453334105 ' lr-AdamW/pg2‚0.00012736006453334105 trainer/global_step‚4099 % _timestamp‚1.7844631656667886e+09 _runtime‚ 2840.05576037 _step‚334‹bÝ.괂 trainer/global_step‚4099 % _timestamp‚1.7844631656667886e+09 _runtime‚ 2840.05576037 _step‚334 ' lr-AdamW/pg1‚0.00012736006453334105 ' lr-AdamW/pg2‚0.00012736006453334105Îxl=2¦- _wandbruntime‚2840  _runtime‚2840mCß'§ÙO R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚321479324515840 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚31105911 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚317716723194678 % _timestamp‚1.7844631656675308e+09 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3381739171840 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚318097691043328 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚318097674003968 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚241725550510592 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚314344262129392 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚31105708 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚32500873 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚318097691043328 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚41246208 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚203 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 trainer/global_step‚4099 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚321479430215168 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚32500871 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚41236065 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚41246208 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚199 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚238243689050624 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚314344245090032 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3372374278860 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚32500871 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚73736936Nôä'¨ÞO KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚41246208 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚199 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚238243689050624 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚314344245090032 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3372374278860 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚32500871 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚73736936 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚321479324515840 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚31105911 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚317716723194678 % _timestamp‚1.7844631656675308e+09 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3381739171840 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚318097691043328 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚318097674003968 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚241725550510592 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚314344262129392 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚31105708 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚32500873 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚318097691043328 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚41246208 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚203 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 trainer/global_step‚4099 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚321479430215168 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚32500871 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚41236065 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0ÏòÔá†2©- _wandbruntime‚2840  _runtime‚2840¦Ì.%ªäM K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚321555259897856 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚31112923 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚184 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚317791797214782 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚41245885 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚31113111 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚73764544 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3382436227584 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚41256042 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 219352064 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚314418535615424 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3382524894720 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚318172840709632 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3373157773460 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚73754385 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3382524894720 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚32508502 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 % _timestamp‚1.7844631660868928e+09 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3482896288256 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3482771045888 _step‚336 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3382436227584 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚73764544 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚314418552654784 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚318172840709632 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚32508500 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚9294604 cODeviceStÒíXÅ atsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚238299826778624 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚318172823670272 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚21818503 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚317791693388884 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3373244559998 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚73754385 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚241782503714816 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚238299732668928 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 trainer/global_step‚4099 _runtime‚2840.475424368 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚32508502 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚318172823670272 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130]Î.#ï&«éM UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3382436227584 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚73764544 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚314418552654784 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚318172840709632 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚32508500 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚9294604 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚238299826778624 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚318172823670272 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚21818503 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚317791693388884 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3373244559998 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚73754385 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚241782503714816 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚238299732668928 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 trainer/global_step‚4099 _runtime‚2840.475424368 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚32508502 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚318172823670272 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚321555259897856 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚31112923 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚184 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚317791797214782 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚41245885 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚31113111 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚73764544 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3382436227584 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚41256042 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 219352064 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚314418535615424 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3382524894720 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚318172840709632 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3373157773460 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚73754385 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3382524894720 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚32508502 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 % _timestamp‚1.7844631660868928e+09 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3482896288256 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3482771045888 _step‚336ÐnMJY2¬- _wandbruntime‚2840  _runtime‚2840|Ã6­þ epoch‚10 0loss/FRt1/assignment_loss‚2.5195175332900805 2loss/SRqqt1/assignment_loss‚0.7286080173266298 2loss/SRqqt2/assignment_loss‚0.6419205857981652 1loss/FBt1/assignment_loss‚0.10964977984924308 0loss/FBt2/assignment_loss‚0.1357750907748966 .loss/FBt2/detection_loss‚1.681188103893111 &loss/total_loss‚11.233639524055935 $ _timestamp‚1.784463166087927e+09 _runtime‚2840.476013829 _step‚337 /loss/FRt2/detection_loss‚1.3560720195850378 0loss/FRt2/assignment_loss‚1.1496029179607707 /loss/FRt1/detection_loss‚0.5850990465133559 1loss/SRqqt1/detection_loss‚1.1120451199723052 0loss/SRqqt2/detection_loss‚0.708400082037525 .loss/FBt1/detection_loss‚0.505761227054813 trainer/global_step‚4099v#½ ®ƒ 0loss/SRqqt2/detection_loss‚0.708400082037525 .loss/FBt1/detection_loss‚0.505761227054813 trainer/global_step‚4099 epoch‚10 0loss/FRt1/assignment_loss‚2.5195175332900805 2loss/SRqqt1/assignment_loss‚0.7286080173266298 2loss/SRqqt2/assignment_loss‚0.6419205857981652 1loss/FBt1/assignment_loss‚0.10964977984924308 0loss/FBt2/assignment_loss‚0.1357750907748966 .loss/FBt2/detection_loss‚1.681188103893111 &loss/total_loss‚11.233639524055935 $ _timestamp‚1.784463166087927e+09 _runtime‚2840.476013829 _step‚337 /loss/FRt2/detection_loss‚1.3560720195850378 0loss/FRt2/assignment_loss‚1.1496029179607707 /loss/FRt1/detection_loss‚0.5850990465133559 1loss/SRqqt1/detection_loss‚1.1120451199723052Ñ4qœx¯‚(:’ ÇþòÒŠÑ⤠gpu.0.gpu‚67.0 gpu.0.memory‚16, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚47.0 gpu.0.powerWatts‚113.44' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚37.81333333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0€P¹j°‚(:ß ÇþòÒ´ø›¥" disk.nvme0c0n1.in‚ 1852.4453125 network.sent‚ 6.062147e+06 disk.dm-0.in‚ 1982.2109375" disk.dm-0.out‚47721.2021484375 disk.rbd0.out‚0# memory_percent‚98.8002061843872) proc.memory.percent‚17.05625057220459 cpu‚25.881148069815435) disk./.usagePercent‚76.33465479763959 disk.md0.out‚69.9375 disk.dm-1.in‚0" disk.dm-1.out‚1849.75537109375! disk.nvme0n1.out‚ 33867.828125 disk.nvme1c1n1.in‚ 129.765625( disk.nvme1c1n1.out‚15853.72509765625 disk.nvme1n1.in‚ 129.765625% disk./.usageGB‚1171.9277877807617 disk.nvme0n1.in‚ 1852.4453125& disk.nvme1n1.out‚15853.72509765625( proc.memory.availableMB‚ 196.57421875# proc.memory.rssMB‚ 2794.49609375 proc.cpu.threads‚17# disk.nvme0c0n1.out‚ 33867.828125 network.recv‚764017 disk.md0.in‚0 disk.rbd0.in‚0—¦Ù@q±‚(:æ ÖþòÒ¤ì̤ network.recv‚768339 disk.nvme0n1.in‚ 1852.4453125& disk.nvme1c1n1.out‚15854.181640625 disk.dm-1.in‚0$ memory_percent‚98.79870414733887' proc.memory.availableMB‚ 196.8203125) proc.memory.percent‚17.05625057220459 disk.md0.out‚ 70.1015625) disk./.usagePercent‚76.33465007663335" disk.nvme0c0n1.in‚ 1852.4453125$ disk.nvme1n1.out‚15854.181640625 disk.dm-0.in‚ 1982.2109375# proc.memory.rssMB‚ 2794.49609375 proc.cpu.threads‚17 network.sent‚ 6.123626e+06& disk.nvme0n1.out‚33869.12548828125# disk.dm-0.out‚47722.44580078125 cpu‚25.88134482516743% disk./.usageGB‚1171.9277153015137( disk.nvme0c0n1.out‚33869.12548828125 disk.nvme1c1n1.in‚ 129.765625 disk.nvme1n1.in‚ 129.765625 disk.md0.in‚0 disk.dm-1.out‚1849.884765625 disk.rbd0.in‚0 disk.rbd0.out‚0êŒàþ“²‚(:ˆ ÖþòÒÜ¥ gpu.0.gpu‚88.0 gpu.0.memory‚21, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚47.0 gpu.0.powerWatts‚115.494' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚38.498 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¦ðë2³- _wandbruntime‚2873  _runtime‚2873eÜŠoÈ´Â ' lr-AdamW/pg2‚0.00012704644328087673 trainer/global_step‚4149 $ _timestamp‚1.784463199448043e+09 _runtime‚2873.836391329 _step‚338 ' lr-AdamW/pg1‚0.00012704644328087673èt굂 _runtime‚2873.836391329 _step‚338 ' lr-AdamW/pg1‚0.00012704644328087673 ' lr-AdamW/pg2‚0.00012704644328087673 trainer/global_step‚4149 $ _timestamp‚1.784463199448043e+09Ò:i@Ã2¶- _wandbruntime‚2873  _runtime‚2873ˆ3áß'·ÙO XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚22066251 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚244573277107200 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚321470384169300 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚74611238 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚31466596 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚241050575450624 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3421000452096 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚41739060 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 150006272 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3411612809668 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 trainer/global_step‚4149 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚32882323 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3522584795648 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚321470487995086 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚32882323 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚9400349 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚186 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3522701656576 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚325276263470592 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚325276157771264 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚318058771359632 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚41728917 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚325276263470592 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚74621383 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚321855157319168 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚321855174358528 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚244573066136576 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚318058788398992 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚321855157319168 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚31466786 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚321855174358528 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚41728917 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 _step‚339 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚241050481340928 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 116860928 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 _runtime‚2873.837615611 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚41739060 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚325276157771264 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚41728917 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 _step‚339 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚241050481340928 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 116860928 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 _runtime‚2873.837615611 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚41739060 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚325276157771264 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚31466596 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚241050575450624 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3421000452096 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚41739060 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 150006272 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3411612809668 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 trainer/global_step‚4149 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚32882323 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3522584795648 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚321470487995086 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚32882323 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚9400349 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚186 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3522701656576 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚325276263470592 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚325276157771264 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚318058771359632 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚41728917 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚325276263470592 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚74621383 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚321855157319168 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0Óïöí2¹- _wandbruntime‚2874  _runtime‚2874¾Áê&ºäM P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3523388598272 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚325352198859776 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚9402704 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚187 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 127339520 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 150006272 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚318133078924384 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3523515937792 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚74628687 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚74638846 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚74638846 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚32889952 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚241106713178624 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚32889952 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚325352198859776 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚241106619068928 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚321930324024832 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚244630229116416 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3412396304268 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3421874834944 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚31474011 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚74628687 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚22071307 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚9402517 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚32889950 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚321545458189292 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3421786167808 trainer/global_step‚4149 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3421786167808 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚321930324024832 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚41748894 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚31473820 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 % _timestamp‚1.7844631998722851e+09 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚41738737 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 221449216 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 _runtime‚2874.260950772 _step‚340 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚41748894 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898Ô)ˆï&»éM XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3412396304268 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3421874834944 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚31474011 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚74628687 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚22071307 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚9402517 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚32889950 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚321545458189292 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3421786167808 trainer/global_step‚4149 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3421786167808 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚321930324024832 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚41748894 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚31473820 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 % _timestamp‚1.7844631998722851e+09 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚41738737 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 221449216 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 _runtime‚2874.260950772 _step‚340 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚41748894 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3523388598272 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚325352198859776 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚9402704 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚187 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 127339520 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 150006272 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚318133078924384 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3523515937792 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚74628687 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚74638846 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚74638846 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚32889952 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚241106713178624 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚32889952 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚325352198859776 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚241106619068928 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚321930324024832 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚244630229116416 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0ÔsoFØ2¼- _wandbruntime‚2874  _runtime‚2874O‰ýĽ 1loss/FBt2/assignment_loss‚0.07629021008078367 /loss/FRt1/detection_loss‚0.5526443504854063 .loss/FRt2/detection_loss‚1.438301461389199 0loss/FRt1/assignment_loss‚2.5302492228386075 2loss/SRqqt1/assignment_loss‚0.6029487960327762 2loss/SRqqt2/assignment_loss‚0.6026879602999803 1loss/FBt1/assignment_loss‚0.08237366942560334 1loss/SRqqt2/detection_loss‚0.6860592707647699 .loss/FBt2/detection_loss‚1.100581279758251 epoch‚10 trainer/global_step‚4149 % _timestamp‚1.7844631998744564e+09 _runtime‚2874.262743295 0loss/FRt2/assignment_loss‚1.1230684653788372 1loss/SRqqt1/detection_loss‚1.1290840107397822 /loss/FBt1/detection_loss‚0.5927997060888188 &loss/total_loss‚10.517088403282816 _step‚341¶r¸Ÿ ¾† 2loss/SRqqt2/assignment_loss‚0.6026879602999803 1loss/FBt1/assignment_loss‚0.08237366942560334 1loss/SRqqt2/detection_loss‚0.6860592707647699 .loss/FBt2/detection_loss‚1.100581279758251 epoch‚10 trainer/global_step‚4149 % _timestamp‚1.7844631998744564e+09 _runtime‚2874.262743295 0loss/FRt2/assignment_loss‚1.1230684653788372 1loss/SRqqt1/detection_loss‚1.1290840107397822 /loss/FBt1/detection_loss‚0.5927997060888188 &loss/total_loss‚10.517088403282816 _step‚341 1loss/FBt2/assignment_loss‚0.07629021008078367 /loss/FRt1/detection_loss‚0.5526443504854063 .loss/FRt2/detection_loss‚1.438301461389199 0loss/FRt1/assignment_loss‚2.5302492228386075 2loss/SRqqt1/assignment_loss‚0.6029487960327762ÕÁ©,¨h¿‚(:Ý åþòÒ“ƒÕ¤) proc.memory.percent‚17.07758903503418 network.sent‚ 6.129644e+06 disk.md0.in‚0 disk.md0.out‚ 70.44921875" disk.nvme0c0n1.in‚ 1852.4453125 disk.nvme0n1.in‚ 1852.4453125# disk.nvme0n1.out‚33871.73046875' proc.memory.availableMB‚ 197.7109375 proc.cpu.threads‚17 network.recv‚771525% disk./.usageGB‚1171.9277153015137 disk.nvme1c1n1.in‚ 129.765625 disk.dm-0.in‚ 1982.2109375" proc.memory.rssMB‚ 2797.9921875) disk./.usagePercent‚76.33465007663335% disk.nvme1c1n1.out‚15854.91015625 disk.rbd0.in‚0 disk.rbd0.out‚0 cpu‚25.881785957069955 disk.dm-1.out‚1849.966796875 disk.nvme1n1.in‚ 129.765625# disk.nvme1n1.out‚15854.91015625! disk.dm-0.out‚47724.943359375 disk.dm-1.in‚0% disk.nvme0c0n1.out‚33871.73046875$ memory_percent‚98.793268203735355©|ÞžÀ‚(:“ åþòÒöÓº¥ gpu.0.gpu‚10.0 gpu.0.memory‚2, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚47.0 gpu.0.powerWatts‚113.328' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚37.775999999999996 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¸@sÁ‚(:è ôþòÒ¸Á·¥ network.sent‚ 6.192823e+06 network.recv‚775713' disk.nvme1c1n1.out‚15856.0380859375 disk.md0.out‚ 70.93359375 disk.dm-0.in‚ 1982.2109375# disk.dm-0.out‚47729.43701171875 disk.rbd0.in‚0) proc.memory.percent‚17.07758903503418 disk.nvme1n1.in‚ 129.765625 disk.rbd0.out‚0 disk.nvme0n1.in‚ 1852.4453125& disk.nvme0n1.out‚33876.33349609375$ memory_percent‚98.79331588745117" proc.memory.rssMB‚ 2797.9921875) disk./.usagePercent‚76.31924593027398% disk./.usageGB‚1171.6912231445312% disk.nvme1n1.out‚15856.0380859375 disk.dm-1.in‚0" disk.nvme0c0n1.in‚ 1852.4453125( disk.nvme0c0n1.out‚33876.33349609375& proc.memory.availableMB‚ 197.703125 cpu‚25.88281106122921 disk.nvme1c1n1.in‚ 129.765625 disk.md0.in‚0! disk.dm-1.out‚1850.1728515625 proc.cpu.threads‚17)ežÂ‚(:“ ôþò҅ᤠgpu.0.gpu‚90.0 gpu.0.memory‚17, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚49.0 gpu.0.powerWatts‚228.884' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚76.29466666666667 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Øú2Ã- _wandbruntime‚2907  _runtime‚2907Æáƒ\ÉÄà ' lr-AdamW/pg1‚0.00012672913371846565 ' lr-AdamW/pg2‚0.00012672913371846565 trainer/global_step‚4199 % _timestamp‚1.7844632332390637e+09 _runtime‚2907.627450045 _step‚342æ³C ÎÅÈ ' lr-AdamW/pg1‚0.00012672913371846565 ' lr-AdamW/pg2‚0.00012672913371846565 trainer/global_step‚4199 % _timestamp‚1.7844632332390637e+09 _runtime‚2907.627450045 _step‚342ÖãAQÇ2Æ- _wandbruntime‚2907  _runtime‚2907HZ ÇÚO TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚42221769 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚42221769 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚321773297629232 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3563123341824 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 trainer/global_step‚4199 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3563259077120 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 150006272 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 % _timestamp‚1.7844632332401187e+09 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚329073096726016 ZHDeviceStatsMonitor.on_train_batc0pýúÕh_start/allocated_bytes.small_pool.freed‚ 3460350392320 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚22319051 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚181 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚325224148969708 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚325612640634368 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚33263773 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚329072991026688 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚75495685 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚42231912 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚31826666 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚247420491082752 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚329072991026688 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚325612657673728 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚243857367740928 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3460439052288 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 135735296 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚185 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 _step‚343 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3460439052288 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 229844992 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0ËOÕÝå'ÈßO UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚329072991026688 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚75495685 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚42231912 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚31826666 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚247420491082752 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚329072991026688 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚325612657673728 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚243857367740928 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3460439052288 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 135735296 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚185 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 _step‚343 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3460439052288 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 229844992 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚42221769 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚42221769 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚321773297629232 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3563123341824 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 trainer/global_step‚4199 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3563259077120 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 150006272 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 % _timestamp‚1.7844632332401187e+09 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚329073096726016 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3460350392320 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚22319051 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚181 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚325224148969708 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚325612640634368 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚33263773׶€ü2É- _wandbruntime‚2908  _runtime‚2908Hr ç&ÊáM WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚31833743 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚177 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚22324107 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚33271400 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚247477434219008 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚247477661959680 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 133630976 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚33271402 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚42231589 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚325687790300672 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 $ _timestamp‚1.784463233659682e+09 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚42241746 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚325687807340032 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚9509640 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3451721621614 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚75502989 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚329148926408704 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 227740672 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚75513148 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3564062381056 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 trainer/global_step‚4199 _runtime‚ 2908.0492573 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 _step‚344 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚329149032115200 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚329149032115200 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚33271402 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚31833920 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚33271400 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3461224775168 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚42231589 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚243913505468928 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3461224775168 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3461136108032 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3461136108032 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 150006272 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚325299326815598 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚325687807340032 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚75513148 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚75502989 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚9509813 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚173 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚75513148 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚75502989 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚9509813 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚173 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚31833743 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚177 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚22324107 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚33271400 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚247477434219008 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚247477661959680 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 133630976 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚33271402 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚42231589 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚325687790300672 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 $ _timestamp‚1.784463233659682e+09 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚42241746 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚325687807340032 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚9509640 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3451721621614 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚75502989 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚329148926408704 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 227740672 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚75513148 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 aODeviceStatsMonitor.on_train_batch_end/inacÆK'âtive_split_bytes.small_pool.allocated‚ 3564062381056 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 trainer/global_step‚4199 _runtime‚ 2908.0492573 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 _step‚344 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚329149032115200 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚329149032115200 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚33271402 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚31833920 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚33271400 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3461224775168 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚42231589 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚243913505468928 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3461224775168 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3461136108032 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3461136108032 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 150006272 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚325299326815598 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚325687807340032 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚250268104133120 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚329370140988928 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚332869824282112 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚328977913770116 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚42724764 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚9614036 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3499700332544 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚332869824282112 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3499700332544 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚325487823898832 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚42714621 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚185 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3603632730112 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚332869929981440 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚33645221 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚329370123949568 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚329370123949568 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 150006272 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3490089871284 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 trainer/global_step‚4249 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚332869929981440 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3499788992512 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚246664348250624 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚325487840938192 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚76359842 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚33645221 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 217262080 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 217262080 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚250268104133120 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚329370140988928 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚332869824282112 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚328977913770116 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚42724764 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚9614036 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3499700332544 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚332869824282112 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3499700332544 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚325487823898832 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚42714621 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚185 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3603632730112 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279Û3Œ× 2Û- _wandbruntime‚2942  _runtime‚2942/8E]ÜãM WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 _step‚348 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3500574715392 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3500486048256 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3500574715392 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚329445290655232 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚4249 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚332945865370624 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚76387450 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚32192886 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚33652850 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚76377291 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 131533824 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚246720485978624 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚33652852 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚246720391868928 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 150006272 _runtime‚2942.896295535 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚329053091616006 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚250325059427328 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚325562114424224 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚42734598 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚42734598 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚188 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚22576903 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚325562131463584 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚329445273615872 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚33652852 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚9615983 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3490960152422 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚32193074 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚329052987790108 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚76377291 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 225643520 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚42724441 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚329445290655232 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚184 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3490873365884 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚332945865370624 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚250324833783808 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚332945759664128 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚9616167 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3500486048256 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚329445273615872 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0¦ûÇRî&ÝèM UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚329445273615872 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚33652852 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚9615983 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3490960152422 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚32193074 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚329052987790108 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚76377291 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 225643520 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚42724441 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚329445290655232 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚184 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3490873365884 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚332945865370624 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚250324833783808 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚332945759664128 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚9616167 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3500486048256 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚329445273615872 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 _step‚348 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3500574715392 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3500486048256 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3500574715392 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚329445290655232 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚4249 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚332945865370624 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚76387450 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚32192886 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚33652850 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚76377291 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 131533824 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚246720485978624 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚33652852 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚246720391868928 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 150006272 _runtime‚2942.896295535 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚329053091616006 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚250325059427328 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚325562114424224 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚42734598 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚42734598 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚188 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚22576903 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚325562131463584 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0ܯ|Ó2Þ- _wandbruntime‚2942  _runtime‚2942* Ïßÿ 2loss/SRqqt1/assignment_loss‚0.6878850885112077 /loss/FBt1/detection_loss‚0.5563965342586968 epoch‚10 % _timestamp‚1.7844632685088425e+09 _runtime‚2942.896947926 /loss/FRt1/detection_loss‚0.5721356890600586 .loss/FRt2/detection_loss‚1.421894303343803 1loss/SRqqt2/detection_loss‚0.6977580660294331 /loss/FBt2/detection_loss‚1.1991111875094216 2loss/SRqqt2/assignment_loss‚0.6413673066216566 1loss/FBt1/assignment_loss‚0.10801997861532475 0loss/FBt2/assignment_loss‚0.1662548930572202 0loss/SRqqt1/detection_loss‚1.073398603278985 &loss/total_loss‚10.789383449488778 trainer/global_step‚4249 _step‚349 /loss/FRt1/assignment_loss‚2.543259290297652 0loss/FRt2/assignment_loss‚1.1219025089053178>Î à„ /loss/FRt1/detection_loss‚0.5721356890600586 .loss/FRt2/detection_loss‚1.421894303343803 1loss/SRqqt2/detection_loss‚0.6977580660294331 /loss/FBt2/detection_loss‚1.1991111875094216 2loss/SRqqt2/assignment_loss‚0.6413673066216566 1loss/FBt1/assignment_loss‚0.10801997861532475 0loss/FBt2/assignment_loss‚0.1662548930572202 0loss/SRqqt1/detection_loss‚1.073398603278985 &loss/total_loss‚10.789383449488778 trainer/global_step‚4249 _step‚349 /loss/FRt1/assignment_loss‚2.543259290297652 0loss/FRt2/assignment_loss‚1.1219025089053178 2loss/SRqqt1/assignment_loss‚0.6878850885112077 /loss/FBt1/detection_loss‚0.5563965342586968 epoch‚10 % _timestamp‚1.7844632685088425e+09 _runtime‚2942.896947926Ý”%KOwá‚(:ì °ÿòÒ­’¥% disk./.usageGB‚1171.6904640197754" disk.nvme0c0n1.in‚ 1852.4453125 cpu‚25.89150146245992 proc.cpu.threads‚17 network.sent‚ 6.280651e+06 network.recv‚790116) disk./.usagePercent‚76.31919648394565( disk.nvme0c0n1.out‚33892.80322265625& disk.nvme0n1.out‚33892.80322265625 disk.md0.in‚0 disk.md0.out‚ 72.3671875" disk.dm-1.out‚1850.52197265625 disk.nvme0n1.in‚ 1852.4453125 disk.nvme1n1.in‚ 129.765625& disk.nvme1n1.out‚15859.86865234375 disk.rbd0.in‚0 disk.rbd0.out‚0$ memory_percent‚98.79000186920166( proc.memory.availableMB‚ 198.24609375" proc.memory.rssMB‚ 2831.2265625 disk.nvme1c1n1.in‚ 129.765625( disk.nvme1c1n1.out‚15859.86865234375 disk.dm-0.in‚ 1982.2109375# disk.dm-0.out‚47746.28466796875 disk.dm-1.in‚0) proc.memory.percent‚17.28043556213379•Q¿â‚(:’ °ÿòÒ¹§¿¥ gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚46.0 gpu.0.powerWatts‚113.073' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚37.690999999999995 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0_/¾_nã‚(:ã ¿ÿòÒ®úͤ network.sent‚ 6.330126e+06$ disk./.usageGB‚1171.929141998291$ disk.nvme0n1.out‚34003.232421875 disk.dm-1.in‚0 disk.nvme0n1.in‚ 1852.4453125 disk.md0.in‚0 disk.dm-0.in‚ 1982.2109375" disk.dm-1.out‚1850.66259765625$ memory_percent‚98.79002571105957# proc.memory.rssMB‚ 2833.22265625) proc.memory.percent‚17.29261875152588 proc.cpu.threads‚17 network.recv‚794187 disk.md0.out‚72.625 disk.rbd0.in‚0 disk.rbd0.out‚0" disk.nvme0c0n1.in‚ 1852.4453125& disk.nvme0c0n1.out‚34003.232421875' proc.memory.availableMB‚ 198.2421875) disk./.usagePercent‚76.33474300591377 disk.nvme1c1n1.in‚ 129.765625( disk.nvme1c1n1.out‚15891.49755859375 disk.nvme1n1.in‚ 129.765625& disk.nvme1n1.out‚15891.49755859375" disk.dm-0.out‚47887.6396484375 cpu‚25.893648769386644[ˉ’ä‚(:‡ ¿ÿòҲå gpu.0.gpu‚47.0 gpu.0.memory‚8, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚48.0 gpu.0.powerWatts‚219.444' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚73.148 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0³˜`A2å- _wandbruntime‚2977  _runtime‚2977y=RÇæÁ & lr-AdamW/pg1‚0.0001260835314201885 & lr-AdamW/pg2‚0.0001260835314201885 trainer/global_step‚4299 % _timestamp‚1.7844633030170157e+09 _runtime‚2977.405437599 _step‚350ªnB4ÌçÆ % _timestamp‚1.7844633030170157e+09 _runtime‚2977.405437599 _step‚350 & lr-AdamW/pg1‚0.0001260835314201885 & lr-AdamW/pg2‚0.0001260835314201885 trainer/global_step‚4299Þ1l,?2è- _wandbruntime‚2977  _runtime‚2977Â[c Ø#éÚO KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚186 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚333127607264768 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚32545974 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚77234144 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 129443840 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3539050272768 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚333127624304128 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3539138932736 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚332731782396310 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚43207473 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚329202350168432 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚34026673 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚22824651 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚34026673 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3644101277184 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚329202367207792 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3529328402092 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3539138932736 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚336666657537536 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚253115465371648 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 % _timestamp‚1.7844633030181093e+09 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚9721137 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚336666763236864 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚34026671 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚332731678570524 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 trainer/global_step‚4299 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚43217616 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 _runtime‚2977.406508221 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚43217616 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚333127607264768 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚43207473 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 _step‚351 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚336666763236864 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚9721323 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚249471140540928 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 M@DeviceStvqEêatsMonitor.on_train_batch_start/inactive_split.all.freed‚32545784 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚249471234650624 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚333127624304128 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3539050272768 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚190 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚22824647 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3644230721024 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚336666657537536®M^å'êßO ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚253115465371648 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 % _timestamp‚1.7844633030181093e+09 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚9721137 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚336666763236864 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚34026671 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚332731678570524 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 trainer/global_step‚4299 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚43217616 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 _runtime‚2977.406508221 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚43217616 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚333127607264768 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚43207473 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 _step‚351 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚336666763236864 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚9721323 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚249471140540928 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚32545784 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚249471234650624 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚333127624304128 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3539050272768 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚190 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚22824647 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3644230721024 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚336666657537536 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚186 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚333127607264768 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚32545974 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚77234144 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 129443840 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3539050272768 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚333127624304128 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3539138932736 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚332731782396310 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚43207473 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚329202350168432 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚34026673 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚22824651 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚34026673 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3644101277184 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚329202367207792 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3529328402092 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3539138932736 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚336666657537536ßEµu2ë- _wandbruntime‚2977  _runtime‚2977¡Hÿê&ìäM PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3539835988480 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 trainer/global_step‚4299 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚43217293 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚336742698626048 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚43227450 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚43217293 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚184 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚22829703 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚77251593 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚32553210 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3539924655616 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3539835988480 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚180 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3530198683230 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚249527372378624 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3645038194688 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚333202773970432 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚329276640693824 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 _step‚352 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚22829707 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚77261752 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚9723503 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚253172410573312 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚2977.833622485 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚43227450 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚77251593 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3539924655616 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚336742698626048 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚32553026 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 150006272 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚336742592919552 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚34034302 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 221449216 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 127339520 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3530111896692 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚34034300 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚332806856416414 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚332806752590516 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚333202756931072 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚77261752 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚329276657733184 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3644910855168 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚34034302 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚34034300 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 % _timestamp‚1.7844633034449034e+09Œ¸”aï&íéM S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚332806752590516 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚333202756931072 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚77261752 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚329276657733184 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3644910855168 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚34034302 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚34034300 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 % _timestamp‚1.7844633034449034e+09 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3539835988480 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 trainer/global_step‚4299 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚43217293 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚336742698626048 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚43227450 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚43217293 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚184 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚22829703 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚77251593 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚32553210 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3539924655616 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3539835988480 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚180 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3530198683230 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚249527372378624 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3645038194688 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚333202773970432 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚329276640693824 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 _step‚352 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚22829707 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚77261752 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚9723503 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚253172410573312 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 _runtime‚2977.833622485 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚43227450 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚77251593 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3539924655616 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚336742698626048 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚32553026 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 150006272 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚336742592919552 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚34034302 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 221449216 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 127339520 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3530111896692 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚34034300 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚332806856416414àÙ‚ª2î- _wandbruntime‚2977  _runtime‚2977®ñAúï /loss/FRt2/detection_loss‚1.5089603592724972 epoch‚10 _step‚353 0loss/FRt1/assignment_loss‚2.5495062683493734 1loss/FBt1/assignment_loss‚0.08734549283092798 /loss/FRt1/detection_loss‚0.5481512459097225 0loss/FRt2/assignment_loss‚1.0965973502598985 2loss/SRqqt2/assignment_loss‚0.5874243231989279 1loss/SRqqt2/detection_loss‚0.7052167838812116 /loss/FBt2/detection_loss‚0.9146904346843562 &loss/total_loss‚10.734408710214003 trainer/global_step‚4299 _runtime‚2977.835072087 2loss/SRqqt1/assignment_loss‚0.7517347407914994 1loss/FBt2/assignment_loss‚0.20429303719297423 0loss/SRqqt1/detection_loss‚1.067180329404779 .loss/FBt1/detection_loss‚0.713308344437836 % _timestamp‚1.7844633034467373e+09Âù#h—ð† .loss/FBt1/detection_loss‚0.713308344437836 % _timestamp‚1.7844633034467373e+09 /loss/FRt2/detection_loss‚1.5089603592724972 epoch‚10 _step‚353 0loss/FRt1/assignment_loss‚2.5495062683493734 1loss/FBt1/assignment_loss‚0.08734549283092798 /loss/FRt1/detection_loss‚0.5481512459097225 0loss/FRt2/assignment_loss‚1.0965973502598985 2loss/SRqqt2/assignment_loss‚0.5874243231989279 1loss/SRqqt2/detection_loss‚0.7052167838812116 /loss/FBt2/detection_loss‚0.9146904346843562 &loss/total_loss‚10.734408710214003 trainer/global_step‚4299 _runtime‚2977.835072087 2loss/SRqqt1/assignment_loss‚0.75173ø[¡qu47407914994 1loss/FBt2/assignment_loss‚0.20429303719297423 0loss/SRqqt1/detection_loss‚1.067180329404779á­Œ6Ovñ‚(:ë ÎÿòÒ£·Ì¥ network.recv‚797728% disk./.usageGB‚1171.9294242858887( disk.nvme0c0n1.out‚34083.18994140625 disk.rbd0.out‚0$ memory_percent‚98.78742694854736( proc.memory.availableMB‚ 198.66796875 proc.cpu.threads‚17 disk.nvme1c1n1.in‚ 129.765625& disk.nvme1n1.out‚15921.73779296875 disk.md0.in‚0 disk.dm-0.in‚ 1982.2109375 disk.dm-1.out‚ 1850.80078125 network.sent‚ 6.348896e+06) disk./.usagePercent‚76.33476139299063 disk.nvme0n1.in‚ 1852.4453125( disk.nvme1c1n1.out‚15921.73779296875 disk.nvme1n1.in‚ 129.765625 disk.dm-1.in‚0 disk.rbd0.in‚0# proc.memory.rssMB‚ 2834.94921875" disk.nvme0c0n1.in‚ 1852.4453125& disk.nvme0n1.out‚34083.18994140625 disk.md0.out‚ 73.0859375" disk.dm-0.out‚47996.7216796875* proc.memory.percent‚17.303156852722168 cpu‚25.895330189315338/\z“ò‚(:ˆ ÎÿòÒ˜Ôݤ gpu.0.gpu‚90.0 gpu.0.memory‚18, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚49.0 gpu.0.powerWatts‚222.033' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚74.011 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¦Ÿrþqó‚(:æ ÝÿòÒ™òÔ¤ disk.rbd0.in‚0& disk.nvme0n1.out‚34094.85791015625( proc.memory.availableMB‚ 198.66796875 network.sent‚ 6.399171e+06) disk./.usagePercent‚76.33476561704883 disk.dm-0.in‚ 1982.2109375" disk.nvme0c0n1.in‚ 1852.4453125 disk.nvme0n1.in‚ 1852.4453125$ memory_percent‚98.78742694854736* proc.memory.percent‚17.303156852722168 network.recv‚801677% disk./.usageGB‚1171.9294891357422% disk.nvme1n1.out‚15923.1005859375 disk.md0.in‚0 disk.md0.out‚73.625 disk.rbd0.out‚0# proc.memory.rssMB‚ 2834.94921875 cpu‚25.895370314582685 disk.nvme1n1.in‚ 129.765625# disk.dm-0.out‚48008.56591796875 disk.dm-1.in‚0 disk.dm-1.out‚ 1850.83984375( disk.nvme0c0n1.out‚34094.85791015625 proc.cpu.threads‚17 disk.nvme1c1n1.in‚ 129.765625' disk.nvme1c1n1.out‚15923.1005859375º«w,žô‚(:“ ÝÿòÒðу¥ gpu.0.gpu‚91.0 gpu.0.memory‚21, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚49.0 gpu.0.powerWatts‚226.235' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚75.41166666666666 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ÏÆLƒ2õ- _wandbruntime‚3009  _runtime‚30095  Èö ' lr-AdamW/pg1‚0.00012575528015015166 ' lr-AdamW/pg2‚0.00012575528015015166 trainer/global_step‚4349 $ _timestamp‚1.784463335375485e+09 _runtime‚3009.763873388 _step‚354\»(kÍ÷Ç _step‚354 ' lr-AdamW/pg1‚0.00012575528015015166 ' lr-AdamW/pg2‚0.00012575528015015166 trainer/global_step‚4349 $ _timestamp‚1.784463335375485e+09 _runtime‚3009.763873388âM2ý2ø- _wandbruntime‚3009  _runtime‚3009 ú¼óß'ùÙO PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚34408123 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚78118591 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚340463490792960 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚34408121 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚9827852 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 _runtime‚3009.767835155 _step‚355 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚43700325 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3578488872960 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3684685076480 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚332916876438032 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚43700325 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3578488872960 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚340463490792960 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚336885107619328 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚182 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚23077451 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚178 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚332916893477392 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚336485547196718 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚78108446 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚34408121 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 $ _timestamp‚1.784463335376398e+09 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚32905121 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚43710468 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3568653719326 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚34408123 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 221456384 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 trainer/global_step‚4349 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3578400212992 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚336885090579968 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 127346688 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚32905303 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚9827674 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚252278026940928 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚340463596492288 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚255962806127104 Bä'úÞO D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚34408123 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚78118591 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚340463490792960 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚34408121 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚9827852 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 _runtime‚3009.767835155 _step‚355 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚43700325 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3578488872960 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3684685076480 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚332916876438032 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚43700325 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3578488872960 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚340463490792960 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚336885107619328 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚182 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚23077451 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚178 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚332916893477392 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚336485547196718 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚78108446 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚34408121 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 $ _timestamp‚1.784463335376398e+09 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚32905121 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚43710468 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3568653719326 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚34408123 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 221456384 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 trainer/global_step‚4349 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3578400212992 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚336885090579968 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 127346688 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚32905303 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚9827674 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚252278026940928 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚340463596492288 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚255962806127104 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚32912241 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚170 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚252334164668928 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚332991166963424 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚336960257285632 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3685374410240 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚78125895 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3579185928704 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚78136054 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚336960240246272 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚43710145 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚340539531881472 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚174 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 _step‚356 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚43720302 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚23082507 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚9829738 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3579274595840 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚43710145 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚252334258778624 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 UIDevœoCWÀiceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚9829908 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚78136054 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚32912415 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚34415752 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3579185928704 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 _runtime‚3010.171425694É;êçï&ýéM TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚170 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚252334164668928 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚332991166963424 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚336960257285632 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3685374410240 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚78125895 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3579185928704 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚78136054 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚336960240246272 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚43710145 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚340539531881472 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚174 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 _step‚356 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚43720302 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚23082507 QEDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚9829738 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3579274595840 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚43710145 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚252334258778624 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚9829908 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚78136054 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚32912415 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚34415752 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3579185928704 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 _runtime‚3010.171425694 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3579274595840 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚34415752 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚32912241ä*;“o2þ- _wandbruntime‚3010  _runtime‚3010Þ™ëzÿ % _timestamp‚1.7844633357838855e+09 _runtime‚3010.171980254 0loss/FRt2/assignment_loss‚1.0939943142864699 1loss/SRqqt1/assignment_loss‚0.651656834327011 1loss/FBt1/assignment_loss‚0.03622038310685936 _step‚357 1loss/FBt2/assignment_loss‚0.06317503950015727 1loss/SRqqt1/detection_loss‚1.0579304079881375 1loss/SRqqt2/detection_loss‚0.7112882911118429 /loss/FBt2/detection_loss‚1.0807618451519654 /loss/FRt1/assignment_loss‚2.519948086004596 2loss/SRqqt2/assignment_loss‚0.6468272923208583 /loss/FRt1/detection_loss‚0.5887996655481136 &loss/total_loss‚10.603061011131988 /loss/FRt2/detection_loss‚1.5301310828487134 /loss/FBt1/detection_loss‚0.6223277689372628 epoch‚10 trainer/global_step‚4349õ…m €† 1loss/SRqqt1/detection_loss‚1.0579304079881375 1loss/SRqqt2/detection_loss‚0.7112882911118429 /loss/FBt2/detection_loss‚1.0807618451519654 /loss/FRt1/assignment_loss‚2.519948086004596 2loss/SRqqt2/assignment_loss‚0.6468272923208583 /loss/FRt1/detection_loss‚0.5887996655481136 &loss/total_loss‚10.603061011131988 /loss/FRt2/detection_loss‚1.5301310828487134 /loss/FBt1/detection_loss‚0.6223277689372628 epoch‚10 trainer/global_step‚4349 % _timestamp‚1.7844633357838855e+09 _runtime‚3010.171980254 0loss/FRt2/assignment_loss‚1.0939943142864699 1loss/SRqqt1/assignment_loss‚0.651656834327011 1loss/FBt1/assignment_loss‚0.03622038310685936 _step‚357 1loss/FBt2/assignment_loss‚0.06317503950015727åÓ¢â“z‚(:ï ìÿòÒšƒ¥" disk.dm-1.out‚1850.92041015625" disk.nvme0c0n1.in‚ 1852.4453125 cpu‚25.893983338038513 proc.cpu.threads‚17 network.sent‚ 6.414936e+06% disk./.usageGB‚1171.9294509887695& disk.nvme1n1.out‚15923.58349609375 disk.md0.in‚0 disk.dm-1.in‚0 disk.rbd0.in‚0 disk.rbd0.out‚0 disk.nvme0n1.in‚ 1852.4453125 disk.nvme1c1n1.in‚ 129.765625( disk.nvme1c1n1.out‚15923.58349609375 disk.md0.out‚ 73.90234375( disk.nvme0c0n1.out‚34095.89306640625$ memory_percent‚98.78764152526855# proc.memory.rssMB‚ 2834.94921875* proc.memory.percent‚17.303156852722168) disk./.usagePercent‚76.33476313230871 disk.nvme1n1.in‚ 129.765625 disk.dm-0.in‚ 1982.2109375& disk.nvme0n1.out‚34095.89306640625' proc.memory.availableMB‚ 198.6328125 network.recv‚805027# disk.dm-0.out‚48009.39404296875ü$£àœ‚‚(:‘ ìÿòÒÉÅ°¥ gpu.0.gpu‚17.0 gpu.0.memory‚4, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚47.0 gpu.0.powerWatts‚113.2' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚37.733333333333334 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ÓEå7oƒ‚(:ä ûÿòÒÏää disk.rbd0.in‚0 disk.rbd0.out‚0& disk.nvme0c0n1.out‚34100.240234375( disk.nvme1c1n1.out‚15927.81689453125 cpu‚25.892789703420792 network.sent‚ 6.464448e+06$ disk./.usageGB‚1171.929500579834 disk.md0.in‚0 disk.dm-0.out‚ 48014.234375$ memory_percent‚98.78766536712646 network.recv‚808780 disk.md0.out‚ 74.265625 disk.dm-1.in‚0 disk.nvme0n1.in‚ 1852.4453125$ disk.nvme0n1.out‚34100.240234375( proc.memory.availableMB‚ 198.62890625# proc.memory.rssMB‚ 2834.94921875 proc.cpu.threads‚17) disk./.usagePercent‚76.33476636247086" disk.dm-1.out‚1853.88916015625" disk.nvme0c0n1.in‚ 1852.4453125 disk.nvme1c1n1.in‚ 129.765625 disk.nvme1n1.in‚ 129.765625& disk.nvme1n1.out‚15927.81689453125* proc.memory.percent‚17.303156852722168 disk.dm-0.in‚ 1982.2109375“$hõ„‚(:’ ûÿòÒÿÍ‹¥ gpu.0.gpu‚22.0 gpu.0.memory‚3, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚48.0 gpu.0.powerWatts‚221.771' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚73.92366666666666 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0†{³Æ2…- _wandbruntime‚3042  _runtime‚3042_„û@ƆÀ & lr-AdamW/pg1‚0.0001254234234991632 & lr-AdamW/pg2‚0.0001254234234991632 trainer/global_step‚4399 % _timestamp‚1.7844633679278228e+09 _runtime‚ 3042.31626508 _step‚358ÒëIˇŠ_step‚358 & lr-AdamW/pg1‚0.0001254234234991632 & lr-AdamW/pg2‚0.0001254234234991632 trainer/global_step‚4399 % _timestamp‚1.7844633679278228e+09 _runtime‚ 3042.31626508æÿ¸2ˆ- _wandbruntime‚3042  _runtime‚3042þEDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚258809936611328 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚33264236 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚258810162262016 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3725023270400 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚181 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 225650688 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3607805463708 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚9933985 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚255084913340928 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚44193177 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚336631402707632 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3607892250134 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚44193177 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚9933808 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚340239311997126 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚177 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚344260429747712 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚340642590934528 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚344260429747712 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3617750153216 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚340642590934528 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚336631419746992 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚34789573 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚344260324048384 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚23330247 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚340239208171340 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3617838813184 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 _step‚359 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚34789571 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 trainer/global_step‚4399 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚44203320 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚340642573895168 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚78992893(“¬ŠßO TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚23330251 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚258809936611328 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚33264236 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚258810162262016 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3725023270400 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚181 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 225650688 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3607805463708 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 WKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚9933985 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚255084913340928 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚44193177 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚336631402707632 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3607892250134 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚44193177 SGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚9933808 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚340239311997126 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚177 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚344260429747712 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚340642590934528 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚344260429747712 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3617750153216 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚340642590934528 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚336631419746992 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚34789573 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 WGDeviceStatsMonitor.on_traiF­ ä9 n_batch_start/requested_bytes.large_pool.peak‚ 11862167552 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚344260324048384 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚23330247 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚340239208171340 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3617838813184 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 _step‚359 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚34789571 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 trainer/global_step‚4399 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚44203320 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚340642573895168 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚78992893 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3617838813184çpøfò2‹- _wandbruntime‚3042  _runtime‚3042…J’Ìê&ŒäM UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚185 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚255141051068928 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚336705693233024 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3608588958308 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚79010356 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚181 trainer/global_step‚4399 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚44202997 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚344336365136896 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚258867106415104 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚344336259430400 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚79000197 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 229837824 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚258866876577280 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 150006272 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚34797200 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 % _timestamp‚1.7844633683350928e+09 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚9936108 _runtime‚3042.723593575 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚340717740600832 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 135728128 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚340314386017230 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚340717723561472 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚44202997 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3618535868928 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚340717740600832 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚33271415 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚255141145178624 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚79010356 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚33271230 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚34797200ïÎËï&éM SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 135728128 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚340314386017230 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚340717723561472 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚44202997 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3618535868928 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚340717740600832 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚33271415 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚255141145178624 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚79010356 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚33271230 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚34797200 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚185 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚255141051068928 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚336705693233024 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3608588958308 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚79010356 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚181 trainer/global_step‚4399 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚44202997 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚344336365136896 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚258867106415104 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚344336259430400 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚79000197 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 229837824 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚258866876577280 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 150006272 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚34797200 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 % _timestamp‚1.7844633683350928e+09 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 UIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚9936108 _runtime‚3042.723593575 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚340717740600832 B&Purity/0FRt2SRqqt*FBt/event_proportion‚0.02457682229578495 0"Purity/0FRt2SRqqt1FBt/SRqqt_purity‚ 0.3671875 9"Purity/1FRt0SRqqt*FBt/event_purity‚0.1291322410106659 +jet/accuracy_2_of_3‚0.48350587487220764 *jet/accuracy_2_of_4‚0.7162234783172607 :"Purity/1FRt*SRqqt1FBt/event_purity‚0.43137869238853455 _step‚362 7 Purity/1FRt*SRqqt2FBt/FBt_purity‚0.9854875206947327 9"Purity/1FRt0SRqqt1FBt/event_purity‚0.5602481961250305 7 Purity/1FRt1SRqqt*FBt/FBt_purity‚0.9906229376792908 @&Purity/2FRt0SRqqt1FBt/event_proportion‚0.0007672990905120969 7"Purity/2FRt1SRqqt0FBt/SRqqt_purity‚0.75511234998703 @&Purity/2FRt1SRqqt2FBt/event_proportion‚2.325148852833081e-05 & Purity/2FRt0SRqqt1FBt/FBt_purity‚1 & Purity/2FRt1SRqqt2FBt/FBt_purity‚1 7 Purity/*FRt1SRqqt*FBt/FBt_purity‚0.9860484600067139 & Purity/1FRt*SRqqt1FBt/FBt_purity‚1 & Purity/2FRt1SRqqt1FBt/FBt_purity‚1 & Purity/2FRt0SRqqt2FBt/FBt_purity‚1 jet/accuracy_1_of_5‚1 particle/accuracy_4_of_5‚1 particle/accuracy_5_of_5‚0 9"Purity/*FRt*SRqqt0FBt/SRqqt_purity‚0.6652052998542786 9"Purity/*FRt*SRqqt1FBt/SRqqt_purity‚0.5676465034484863 7 Purity/1FRt0SRqqt2FBt/FRt_purity‚0.9015306830406189 =&Purity/*FRt0SRqqt1FBt/event_proportion‚0.0418526791036129 9"Purity/0FRt0SRqqt1FBt/event_purity‚0.9984753131866455 & Purity/1FRt2SRqqt1FBt/FRt_purity‚0 +jet/accuracy_3_of_3‚0.06548494100570679 +particle/specificity‚0.9098367094993591 7 Purity/0FRt1SRqqt2FBt/FBt_purity‚0.9230769276618958 7 Purity/*FRt*SRqqt1FBt/FBt_purity‚0.9970881938934326 8 Purity/*FRt0SRqqt0FBt/FRt_purity‚0.11580312997102737 =&Purity/*FRt1SRqqt1FBt/event_proportion‚0.0188569575548172 9"Purity/0FRt1SRqqt*FBt/SRqqt_purity‚0.6127545833587646 >&Purity/1FRt*SRqqt1FBt/event_proportion‚0.01643880270421505 ;"Purity/1FRt1SRqqt*FBt/event_purity‚0.030215734615921974 9"Purity/1FRt2SRqqt0FBt/SRqqt_purity‚0.6891425251960754 ,&Purity/1FRt2SRqqt2FBt/event_proportion‚0 +jet/accuracy_2_of_2‚0.12510187923908234 7 Purity/*FRt1SRqqt2FBt/FBt_purity‚0.8796296119689941 9"Purity/*FRt2SRqqt0FBt/event_purity‚0.1454188972711563 7"Purity/0FRt0SRqqt2FBt/event_purity‚0.97723388671875 :"Purity/1FRt*SRqqt*FBt/event_purity‚0.10022103041410446 =&Purity/1FRt*SRqqt0FBt/event_proportion‚0.6105840802192688 & Purity/1FRt1SRqqt1FBt/FBt_purity‚1 A&Purity/1FRt2SRqqt1FBt/event_proportion‚0.00027901786961592734 & Purity/2FRt0SRqqt*FBt/FBt_purity‚1 =&Purity/*FRt0SRqqt*FBt/event_proportion‚0.5129278302192688 9"Purity/0FRt*SRqqt*FBt/SRqqt_purity‚0.6028573513031006 >&Purity/2FRt0SRqqt0FBt/event_proportion‚0.01990327425301075 jet/accuracy_2_of_5‚1 ("Purity/*FRt2SRqqt1FBt/event_purity‚0 7 Purity/0FRt1SRqqt*FBt/FBt_purity‚0.9843019843101501 ("Purity/2FRt1SRqqt*FBt/event_purity‚0 9"Purity/1FRt*SRqqt*FBt/SRqqt_purity‚0.7259965538978577 particle/accuracy_1_of_4‚1 (particle/accuracy‚0.8458271026611328 7 Purity/*FRt*SRqqt*FBt/FBt_purity‚0.9936726093292236 8 Purity/*FRt0SRqqt*FBt/FRt_purity‚0.13168784976005554 9"Purity/0FRt1SRqqt*FBt/event_purity‚0.6098980903625488 >&Purity/1FRt0SRqqt1FBt/event_proportion‚0.00969587080180645 9"Purity/1FRt1SRqqt*FBt/SRqqt_purity‚0.7388723492622375 9"Purity/1FRt1SRqqt1FBt/SRqqt_purity‚0.6537140607833862 7 Purity/*FRt1SRqqt1FBt/FBt_purity‚0.9938595294952393 6 Purity/1FRt*SRqqt*FBt/FBt_purity‚0.995812177658081 7 Purity/1FRt1SRqqt2FBt/FBt_purity‚0.8333333134651184 9"Purity/2FRt1SRqqt*FBt/SRqqt_purity‚0.7496339082717896 & Purity/2FRt1SRqqt*FBt/FBt_purity‚1 8"Purity/0FRt*SRqqt*FBt/event_purity‚0.637143611907959 9"Purity/2FRt*SRqqt1FBt/SRqqt_purity‚0.7037036418914795 9"Purity/0FRt*SRqqt0FBt/event_purity‚0.5869841575622559 6 Purity/0FRt0SRqqt2FBt/FBt_purity‚0.988616943359375 9"Purity/1FRt1SRqqt0FBt/SRqqt_purity‚0.7439113259315491 particle/accuracy_2_of_2‚1 + Purity/2FRt*SRqqt2FBt/FRt_purity‚0.8125 9"Purity/*FRt*SRqqt1FBt/event_purity‚0.7197037935256958 8 Purity/*FRt1SRqqt*FBt/FRt_purity‚0.11535468697547913 8 Purity/1FRt*SRqqt0FBt/FRt_purity‚0.10801371186971664 >&Purity/1FRt2SRqqt*FBt/event_proportion‚0.02322823740541935 9"Purity/2FRt*SRqqt*FBt/SRqqt_purity‚0.7567552328109741 7 Purity/2FRt1SRqqt0FBt/FRt_purity‚0.1363796591758728 9"Purity/2FRt2SRqqt0FBt/SRqqt_purity‚0.7704119086265564 particle/accuracy_1_of_5‚1 8 Purity/*FRt*SRqqt0FBt/FRt_purity‚0.10869201272726059 7 Purity/*FRt0SRqqt1FBt/FBt_purity‚0.9988323450088501 =&Purity/0FRt*SRqqt1FBt/event_proportion‚0.0436662957072258 =&Purity/1FRt0SRqqt0FBt/event_proportion‚0.4408714771270752 @&Purity/2FRt*SRqqt2FBt/event_proportion‚0.0002092633949359879 =&Purity/0FRt0SRqqt1FBt/event_proportion‚0.0313895083963871 :"Purity/1FRt*SRqqt0FBt/event_purity‚0.08875848352909088 ;"Purity/2FRt*SRqqt*FBt/event_purity‚0.012772801332175732 ;"Purity/2FRt0SRqqt0FBt/event_purity‚0.008040143176913261 9"Purity/0FRt*SRqqt2FBt/SRqqt_purity‚0.1538461595773697 ("Purity/0FRt1SRqqt2FBt/event_purity‚0 8"Purity/*FRt0SRqqt1FBt/event_purity‚0.883771538734436 7 Purity/*FRt0SRqqt2FBt/FBt_purity‚0.9908071160316467 =&Purity/0FRt1SRqqt0FBt/even|·Eæ$t_proportion‚0.2083565890789032 ("Purity/0FRt2SRqqt1FBt/event_purity‚0 6 Purity/1FRt0SRqqt1FBt/FRt_purity‚0.618099570274353 particle/accuracy_1_of_2‚1 ,"Purity/2FRt*SRqqt2FBt/event_purity‚0.625 :"Purity/*FRt0SRqqt*FBt/event_purity‚0.19061869382858276 9"Purity/0FRt*SRqqt2FBt/event_purity‚0.9201552271842957 9"Purity/1FRt*SRqqt1FBt/SRqqt_purity‚0.6345606446266174 8 Purity/1FRt1SRqqt*FBt/FRt_purity‚0.10845821350812912 =&Purity/1FRt1SRqqt0FBt/event_proportion‚0.1467633992433548 8 Purity/2FRt0SRqqt0FBt/FRt_purity‚0.11267189681529999 9"Purity/0FRt1SRqqt2FBt/SRqqt_purity‚0.1538461595773697 =&Purity/*FRt0SRqqt0FBt/event_proportion‚0.4607747495174408 7 Purity/0FRt0SRqqt1FBt/FBt_purity‚0.9984753131866455 ?&Purity/0FRt1SRqqt1FBt/event_proportion‚0.011555989272892475 9"Purity/2FRt*SRqqt0FBt/SRqqt_purity‚0.7598997354507446 ("Purity/2FRt*SRqqt2FBt/SRqqt_purity‚0 8 Purity/*FRt*SRqqt*FBt/FRt_purity‚0.12422997504472733 ?&Purity/*FRt*SRqqt2FBt/event_proportion‚0.010997953824698925 7 Purity/*FRt0SRqqt*FBt/FBt_purity‚0.9962381720542908 =&Purity/*FRt1SRqqt0FBt/event_proportion‚0.3670479953289032 & Purity/*FRt2SRqqt1FBt/FRt_purity‚0 @&Purity/0FRt2SRqqt1FBt/event_proportion‚0.0007207961170934141 9"Purity/1FRt1SRqqt1FBt/event_purity‚0.2523530423641205 +particle/sensitivity‚0.6117823719978333 :"Purity/*FRt0SRqqt0FBt/event_purity‚0.11145269125699997 $ _timestamp‚1.784463381945345e+09 9"Purity/*FRt0SRqqt2FBt/event_purity‚0.9158361554145813 8 Purity/*FRt1SRqqt0FBt/FRt_purity‚0.09745977073907852 9"Purity/0FRt2SRqqt0FBt/event_purity‚0.3264159560203552 8 Purity/1FRt*SRqqt*FBt/FRt_purity‚0.12305840849876404 :"Purity/2FRt*SRqqt1FBt/event_purity‚0.13833333551883698 8 Purity/2FRt1SRqqt*FBt/FRt_purity‚0.15498526394367218 /particle/accuracy_4_of_4‚0.9491631388664246 :"Purity/*FRt1SRqqt0FBt/event_purity‚0.35866522789001465 8 Purity/1FRt1SRqqt0FBt/FRt_purity‚0.09092508256435394 8 Purity/2FRt0SRqqt*FBt/FRt_purity‚0.13743048906326294 jet/accuracy_5_of_5‚0 7 Purity/0FRt0SRqqt*FBt/FBt_purity‚0.9953533411026001 9"Purity/1FRt0SRqqt2FBt/event_purity‚0.7530612349510193 ,&Purity/2FRt2SRqqt2FBt/event_proportion‚0 particle/accuracy_2_of_5‚1 =&Purity/*FRt1SRqqt*FBt/event_proportion‚0.3866024911403656 =&Purity/0FRt0SRqqt0FBt/event_proportion‚0.0462472103536129 @&Purity/1FRt1SRqqt1FBt/event_proportion‚0.0064639137126505375 <&Purity/2FRt*SRqqt*FBt/event_proportion‚0.040062315762043 8"Purity/*FRt2SRqqt1FBt/SRqqt_purity‚0.347222238779068 ?&Purity/0FRt0SRqqt2FBt/event_proportion‚0.007626487873494625 =&Purity/2FRt0SRqqt*FBt/event_proportion‚0.0208565853536129 9"Purity/*FRt*SRqqt0FBt/event_purity‚0.2163497656583786 ?&Purity/0FRt*SRqqt2FBt/event_proportion‚0.008068266324698925 7 Purity/1FRt0SRqqt*FBt/FBt_purity‚0.9988385438919067 8 Purity/2FRt*SRqqt*FBt/FRt_purity‚0.13340194523334503 ) Purity/*FRt1SRqqt2FBt/FRt_purity‚0.75 7 Purity/0FRt*SRqqt*FBt/FBt_purity‚0.9926229119300842 9"Purity/0FRt1SRqqt1FBt/SRqqt_purity‚0.5546547770500183 'particle/f_score‚0.6301988363265991 _runtime‚3056.334208147 9"Purity/*FRt1SRqqt1FBt/SRqqt_purity‚0.5910544395446777 =&Purity/0FRt*SRqqt*FBt/event_proportion‚0.2839471697807312 =&Purity/1FRt*SRqqt*FBt/event_proportion‚0.6297432780265808 /particle/accuracy_3_of_4‚0.9850907325744629 :"Purity/1FRt2SRqqt1FBt/SRqqt_purity‚0.32499998807907104 9"Purity/1FRt*SRqqt2FBt/SRqqt_purity‚0.2222222238779068 =&Purity/0FRt1SRqqt*FBt/event_proportion‚0.2203543484210968 9"Purity/0FRt1SRqqt0FBt/SRqqt_purity‚0.6171101331710815 >&Purity/0FRt2SRqqt0FBt/event_proportion‚0.02385602705180645 particle/accuracy_2_of_4‚1 9"Purity/1FRt*SRqqt2FBt/event_purity‚0.6834467649459839 7 Purity/1FRt1SRqqt2FBt/FRt_purity‚0.7777777910232544 9 Purity/1FRt2SRqqt*FBt/FRt_purity‚0.058650702238082886 @&Purity/2FRt*SRqqt1FBt/event_proportion‚0.0016043527284637094 ("Purity/2FRt1SRqqt0FBt/event_purity‚0 =&Purity/*FRt*SRqqt*FBt/event_proportion‚0.9537528157234192 =&Purity/*FRt0SRqqt2FBt/event_proportion‚0.0103004090487957 7 Purity/*FRt2SRqqt1FBt/FBt_purity‚0.9814814329147339 :"Purity/1FRt0SRqqt0FBt/event_purity‚0.11614324897527695 :"Purity/2FRt0SRqqt*FBt/event_purity‚0.02443585731089115 ,jet/accuracy_4_of_4‚0.013899499550461769 7 Purity/*FRt*SRqqt2FBt/FRt_purity‚0.8864323496818542 9"Purity/*FRt1SRqqt*FBt/event_purity‚0.3597860336303711 :"Purity/*FRt2SRqqt*FBt/event_purity‚0.14273817837238312 @&Purity/0FRt1SRqqt2FBt/event_proportion‚0.0004417782765813172 ("Purity/2FRt1SRqqt1FBt/event_purity‚0 9"Purity/2FRt0SRqqt1FBt/event_purity‚0.2864583432674408 9"Purity/*FRt*SRqqt*FBt/event_purity‚0.2563784420490265 @&Purity/*FRt1SRqqt2FBt/event_proportion‚0.0006975446594879031 9"Purity/0FRt2SRqqt*FBt/event_purity‚0.3172382116317749 *jet/accuracy_1_of_1‚0.2804163098335266 *loss/val_total_loss‚10.313489241276079 & Purity/1FRt0SRqqt1FBt/FBt_purity‚1 @&Purity/1FRt1SRqqt2FBt/event_proportion‚0.0002325148816453293 8 Purity/2FRt*SRqqt0FBt/FRt_purity‚0.11400049179792404 9"Purity/0FRt*SRqqt1FBt/event_purity‚0.8498997688293457 7 Purity/0FRt*SRqqt1FBt/FBt_purity‚0.9959661364555359 7 Purity/1FRt*SRqqt2FBt/FRt_purity‚0.8859410881996155 ("Purity/1FRt2SRqqt*FBt/event_purity‚0 =&Purity/2FRt*SRqqt0FBt/event_proportion‚0.0382486991584301 ( Purity/2FRt1SRqqt2FBt/FRt_purity‚0.5 9"Purity/*FRt*SRqqt*FBt/SRqqt_purity‚0.6603255271911621 7"Purity/*FRt2SRqqt0FBt/SRqqt_purity‚0.64164799451828 7 Purity/0FRt*SRqqt2FBt/FBt_purity‚0.9840921759605408 9"Purity/0FRt2SRqqt*FBt/SRqqt_purity‚0.5600875020027161 @&Purity/1FRt*SRqqt2FBt/event_proportion‚0.0027204242069274187 7 Purity/1FRt2SRqqt0FBt/FRt_purity‚0.0594051294028759 /particle/accuracy_3_of_3‚0.9994933605194092 7 Purity/2FRt0SRqqt2FBt/FRt_purity‚0.8571428656578064 7 Purity/*FRt0SRqqt1FBt/FRt_purity‚0.6132236123085022 8"Purity/*FRt1SRqqt*FBt/SRqqt_purity‚0.667289137840271 9"Purity/*FRt2SRqqt*FBt/SRqqt_purity‚0.6362565159797668 7 Purity/*FRt2SRqqt*FBt/FBt_purity‚0.9814814329147339 =&Purity/1FRt0SRqqt*FBt/event_proportion‚0.4530552327632904 7 Purity/1FRt0SRqqt2FBt/FBt_purity‚0.9970238208770752 ("Purity/1FRt2SRqqt0FBt/event_purity‚0 8&Purity/1FRt2SRqqt0FBt/event_proportion‚ 0.02294921875 9"Purity/1FRt2SRqqt*FBt/SRqqt_purity‚0.6843730211257935 *jet/accuracy_1_of_3‚0.9112595915794373 *jet/accuracy_1_of_4‚0.9649112224578857 9"Purity/2FRt0SRqqt2FBt/event_purity‚0.7142857313156128 particle/accuracy_1_of_3‚1 trainer/global_step‚4399 A&Purity/2FRt0SRqqt2FBt/event_proportion‚0.00018601190822664648 <&Purity/*FRt*SRqqt0FBt/event_proportion‚0.881045401096344 8 Purity/*FRt1SRqqt1FBt/FRt_purity‚0.46795904636383057 =&Purity/0FRt*SRqqt0FBt/event_proportion‚0.2322126179933548 =&Purity/0FRt0SRqqt*FBt/event_proportion‚0.0390159972012043 9"Purity/1FRt*SRqqt0FBt/SRqqt_purity‚0.7306254506111145 ("Purity/1FRt1SRqqt2FBt/event_purity‚0 jet/accuracy_3_of_5‚1 7 Purity/*FRt2SRqqt0FBt/FRt_purity‚0.0662318766117096 ("Purity/0FRt0SRqqt0FBt/event_purity‚1 <"Purity/2FRt*SRqqt0FBt/event_purity‚0.0042312657460570335 =&Purity/2FRt1SRqqt*FBt/event_proportion‚0.0127883180975914 ?&Purity/2FRt2SRqqt*FBt/event_proportion‚0.006417410913854837 particle/accuracy_3_of_5‚1 <&Purity/*FRt2SRqqt*FBt/event_proportion‚0.054222472012043 9"Purity/1FRt1SRqqt2FBt/SRqqt_purity‚0.2222222238779068 & Purity/1FRt2SRqqt*FBt/FBt_purity‚1 ,&Purity/2FRt2SRqqt1FBt/event_proportion‚0 particle/accuracy_1_of_1‚1 =&Purity/1FRt1SRqqt*FBt/event_proportion‚0.1534598171710968 ("Purity/2FRt2SRqqt0FBt/event_purity‚0 +jet/accuracy_3_of_4‚0.07885383069515228 ("Purity/2FRt1SRqqt2FBt/SRqqt_purity‚0 ,&Purity/*FRt2SRqqt2FBt/event_proportion‚0 9"Purity/0FRt*SRqqt0FBt/SRqqt_purity‚0.6073150038719177 9"Purity/0FRt0SRqqt*FBt/event_purity‚0.9944429993629456 9"Purity/0FRt1SRqqt1FBt/event_purity‚0.5084317326545715 ,&Purity/0FRt2SRqqt2FBt/event_proportion‚0 7 Purity/1FRt0SRqqt*FBt/FRt_purity‚0.1312306672334671 :"Purity/1FRt1SRqqt0FBt/event_purity‚0.02052449807524681 & Purity/2FRt*SRqqt*FBt/FBt_purity‚1 9"Purity/*FRt1SRqqt1FBt/event_purity‚0.3963554799556732 8 Purity/1FRt1SRqqt1FBt/FRt_purity‚0.46628493070602417 7 Purity/2FRt2SRqqt*FBt/FRt_purity‚0.0762423649430275 ("Purity/2FRt1SRqqt2FBt/event_purity‚0 , Purity/0FRt2SRqqt*FBt/FBt_purity‚0.96875 ("Purity/1FRt2SRqqt1FBt/event_purity‚0 jet/accuracy_4_of_5‚0 7 Purity/*FRt*SRqqt1FBt/FRt_purity‚0.5455740094184875 :"Purity/*FRt1SRqqt2FBt/SRqqt_purity‚0.18518517911434174 8 Purity/*FRt2SRqqt*FBt/FRt_purity‚0.06568708270788193 & Purity/2FRt*SRqqt2FBt/FBt_purity‚1 epoch‚10 8"Purity/*FRt1SRqqt0FBt/SRqqt_purity‚0.672247052192688 & Purity/1FRt2SRqqt1FBt/FBt_purity‚1 ("Purity/2FRt2SRqqt*FBt/event_purity‚0 ("Purity/*FRt1SRqqt2FBt/event_purity‚0 @&Purity/*FRt2SRqqt1FBt/event_proportion‚0.0009998140158131719 7 Purity/1FRt*SRqqt1FBt/FRt_purity‚0.5533981919288635 8 Purity/2FRt1SRqqt1FBt/FRt_purity‚0.44675928354263306 particle/accuracy_2_of_3‚1 :"Purity/*FRt*SRqqt2FBt/SRqqt_purity‚0.18518517911434174 9"Purity/2FRt1SRqqt1FBt/SRqqt_purity‚0.7037036418914795 9"Purity/2FRt2SRqqt*FBt/SRqqt_purity‚0.7704119086265564 *jet/accuracy_1_of_2‚0.7681604027748108ˆÖÓ=•Í{ ("Purity/*FRt2SRqqt1FBt/event_purity‚0 7 Purity/0FRt1SRqqt*FBt/FBt_purity‚0.9843019843101501 ("Purity/2FRt1SRqqt*FBt/event_purity‚0 9"Purity/1FRt*SRqqt*FBt/SRqqt_purity‚0.7259965538978577 particle/accuracy_1_of_4‚1 (particle/accuracy‚0.8458271026611328 7 Purity/*FRt*SRqqt*FBt/FBt_purity‚0.9936726093292236 8 Purity/*FRt0SRqqt*FBt/FRt_purity‚0.13168784976005554 9"Purity/0FRt1SRqqt*FBt/event_purity‚0.6098980903625488 >&Purity/1FRt0SRqqt1FBt/event_proportion‚0.00969587080180645 9"Purity/1FRt1SRqqt*FBt/SRqqt_purity‚0.7388723492622375 9"Purity/1FRt1SRqqt1FBt/SRqqt_purity‚0.6537140607833862 7 Purity/*FRt1SRqqt1FBt/FBt_purity‚0.9938595294952393 6 Purity/1FRt*SRqqt*FBt/FBt_purity‚0.995812177658081 7 Purity/1FRt1SRqqt2FBt/FBt_purity‚0.8333333134651184 9"Purity/2FRt1SRqqt*FBt/SRqqt_purity‚0.7496339082717896 & Purity/2FRt1SRqqt*FBt/FBt_purity‚1 8"Purity/0FRt*SRqqt*FBt/event_purity‚0.637143611907959 9"Purity/2FRt*SRqqt1FBt/SRqqt_purity‚0.7037036418914795 9"Purity/0FRt*SRqqt0FBt/event_purity‚0.5869841575622559 6 Purity/0FRt0SRqqt2FBt/FBt_purity‚0.988616943359375 9"Purity/1FRt1SRqqt0FBt/SRqqt_purity‚0.7439113259315491 particle/accuracy_2_of_2‚1 + Purity/2FRt*SRqqt2FBt/FRt_purity‚0.8125 9"Purity/*FRt*SRqqt1FBt/event_purity‚0.7197037935256958 8 Purity/*FRt1SRqqt*FBt/FRt_purity‚0.11535468697547913 8 Purity/1FRt*SRqqt0FBt/FRt_purity‚0.10801371186971664 >&Purity/1FRt2SRqqt*FBt/event_proportion‚0.02322823740541935 9"Purity/2FRt*SRqqt*FBt/SRqqt_purity‚0.7567552328109741 7 Purity/2FRt1SRqqt0FBt/FRt_purity‚0.1363796591758728 9"Purity/2FRt2SRqqt0FBt/SRqqt_purity‚0.7704119086265564 particle/accuracy_1_of_5‚1 8 Purity/*FRt*SRqqt0FBt/FRt_purity‚0.10869201272726059 7 Purity/*FRt0SRqqt1FBt/FBt_purity‚0.9988323450088501 =&Purity/0FRt*SRqqt1FBt/event_proportion‚0.0436662957072258 =&Purity/1FRt0SRqqt0FBt/event_proportion‚0.4408714771270752 @&Purity/2FRt*SRqqt2FBt/event_proportion‚0.0002092633949359879 =&Purity/0FRt0SRqqt1FBt/event_proportion‚0.0313895083963871 :"Purity/1FRt*SRqqt0FBt/event_purity‚0.08875848352909088 ;"Purity/2FRt*SRqqt*FBt/event_purity‚0.012772801332175732 ;"Purity/2FRt0SRqqt0FBt/event_purity‚0.008040143176913261 9"Purity/0FRt*SRqqt2FBt/SRqqt_purity‚0.1538461595773697 ("Purity/0FRt1SRqqt2FBt/event_purity‚0 8"Purity/*FRt0SRqqt1FBt/event_purity‚0.883771538734436 7 Purity/*FRt0SRqqt2FBt/FBt_purity‚0.9908071160316467 =&Purity/0FRt1SRqqt0FBt/event_proportion‚0.2083565890789032 ("Purity/0FRt2SRqqt1FBt/event_purity‚0 6 Purity/1FRt0SRqqt1FBt/FRt_purity‚0.618099570274353 particle/accuracy_1_of_2‚1 ,"Purity/2FRt*SRqqt2FBt/event_purity‚0.625 :"Purity/*FRt0SRqqt*FBt/event_purity‚0.19061869382858276 9"Purity/0FRt*SRqqt2FBt/event_purity‚0.9201552271842957 9"Purity/1FRt*SRqqt1FBt/SRqqt_purity‚0.6345606446266174 8 Purity/1FRt1SRqqt*FBt/FRt_purity‚0.10845821350812912 =&Purity/1FRt1SRqqt0FBt/event_proportion‚0.1467633992433548 8 Purity/2FRt0SRqqt0FBt/FRt_purity‚0.11267189681529999 9"Purity/0FRt1SRqqt2FBt/SRqqt_purity‚0.1538461595773697 =&Purity/*FRt0SRqqt0FBt/event_proportion‚0.4607747495174408 7 Purity/0FRt0SRqqt1FBt/FBt_purity‚0.9984753131866455 ?&Purity/0FRt1SRqqt1FBt/event_proportion‚0.011555989272892475 9"Purity/2FRt*SRqqt0FBt/SRqqt_purity‚0.7598997354507446 ("Purity/2FRt*SRqqt2FBt/SRqqt_purity‚0 8 Purity/*FRt*SRqqt*FBt/FRt_purity‚0.12422997504472733 ?&Purity/*FRt*SRqqt2FBt/event_proportion‚0.010997953824698925 7 Purity/*FRt0SRqqt*FBt/FBt_purity‚0.9962381720542908 =&Purity/*FRt1SRqqt0FBt/event_proportion‚0.3670479953289032 & Purity/*FRt2SRqqt1FBt/FRt_purity‚0 @&Purity/0FRt2SRqqt1FBt/event_proportion‚0.0007207961170934141 9"Purity/1FRt1SRqqt1FBt/event_purity‚0.2523530423641205 +particle/sensitivity‚0.6117823719978333 :"Purity/*FRt0SRqqt0FBt/event_purity‚0.11145269125699997 $ _timestamp‚1.784463381945345e+09 9"Purity/*FRt0SRqqt2FBt/event_purity‚0.9158361554145813 8 Purity/*FRt1SRqqt0FBt/FRt_purity‚0.09745977073907852 9"Purity/0FRt2SRqqt0FBt/event_purity‚0.3264159560203552 8 Purity/1FRt*SRqqt*FBt/FRt_purity‚0.12305840849876404 :"Purity/2FRt*SRqqt1FBt/event_purity‚0.13833333551883698 8 Purity/2FRt1SRqqt*FBt/FRt_purity‚0.15498526394367218 /particle/accuracy_4_of_4‚0.9491631388664246 :"Purity/*FRt1SRqqt0FBt/event_purity‚0.35866522789001465 8 Purity/1FRt1SRqqt0FBt/FRt_purity‚0.09092508256435394 8 Purity/2FRt0SRqqt*FBt/FRt_purity‚0.13743048906326294 jet/accuracy_5_of_5‚0 7 Purity/0FRt0SRqqt*FBt/FBt_purity‚0.9953533411026001 9"Purity/1FRt0SRqqt2FBt/event_purity‚0.7530612349510193 ,&Purity/2FRt2SRqqt2FBt/event_proportion‚0 particle/accuracy_2_of_5‚1 =&Purity/*FRt1SRqqt*FBt/event_proportion‚0.3866024911403656 =&Purity/0FRt0SRqqt0FBt/event_proportion‚0.0462472103536129 @&Purity/1FRt1SRqqt1FBt/event_proportion‚0.0064639137126505375 <&Purity/2FRt*SRqqt*FBt/event_proportion‚0.040062315762043 8"Purity/*FRt2SRqqt1FBt/SRqqt_purity‚0.347222238779068 ?&Purity/0FRt0SRqqt2FBt/event_proportion‚0.007626487873494625 =&Purity/2FRt0SRqqt*FBt/event_proportion‚0.0208565853536129 9"Purity/*FRt*SRqqt0FBt/event_purity‚0.2163497656583786 ?&Purity/0FRt*SRqqt2FBt/event_proportion‚0.008068266324698925 7 Purity/1FRt0SRqqt*FBt/FBt_purity‚0.9988385438919067 8 Purity/2FRt*SRqqt*FBt/FRt_purity‚0.13340194523334503 ) Purity/*FRt1SRqqt2FBt/FRt_purity‚0.75 7 Purity/0FRt*SRqqt*FBt/FBt_purity‚0.9926229119300842 9"Purity/0FRt1SRqqt1FBt/SRqqt_purity‚0.5546547770500183 'particle/f_score‚0.6301988363265991 _runtime‚3056.334208147 9"Purity/*FRt1SRqqt1FBt/SRqqt_purity‚0.5910544395446777 =&Purity/0FRt*SRqqt*FBt/event_proportion‚0.2839471697807312 =&Purity/1FRt*SRqqt*FBt/event_proportion‚0.6297432780265808 /particle/accuracy_3_of_4‚0.9850907325744629 :"Purity/1FRt2SRqqt1FBt/SRqqt_purity‚0.32499998807907104 9"Purity/1FRt*SRqqt2FBt/SRqqt_purity‚0.2222222238779068 =&Purity/0FRt1SRqqt*FBt/event_proportion‚0.2203543484210968 9"Purity/0FRt1SRqqt0FBt/SRqqt_purity‚0.6171101331710815 >&Purity/0FRt2SRqqt0FBt/event_proportion‚0.02385602705180645 particle/accuracy_2_of_4‚1 9"Purity/1FRt*SRqqt2FBt/event_purity‚0.6834467649459839 7 Purity/1FRt1SRqqt2FBt/FRt_purity‚0.7777777910232544 9 Purity/1FRt2SRqqt*FBt/FRt_purity‚0.058650702238082886 @&Purity/2FRt*SRqqt1FBt/event_proportion‚0.0016043527284637094 ("Purity/2FRt1SRqqt0FBt/event_purity‚0 =&Purity/*FRt*SRqqt*FBt/event_proportion‚0.9537528157234192 =&Purity/*FRt0SRqqt2FBt/event_proportion‚0.0103004090487957 7 Purity/*FRt2SRqqt1FBt/FBt_purity‚0.9814814329147339 :"Purity/1FRt0SRqqt0FBt/event_purity‚0.11614324897527695 :"Purity/2FRt0SRqqt*FBt/event_purity‚0.02443585731089115 ,jet/accuracy_4_of_4‚0.013899499550461769 7 Purity/*FRt*SRqqt2FBt/FRt_purity‚0.8864323496818542 9"Purity/*FRt1SRqqt*FBt/event_purity‚0.3597860336303711 :"Purity/*FRt2SRqqt*FBt/event_purity‚0.14273817837238312 @&Purity/0FRt1SRqqt2FBt/event_proportion‚0.0004417782765813172 ("Purity/2FRt1SRqqt1FBt/event_purity‚0 9"Purity/2FRt0SRqqt1FBt/event_purity‚0.2864583432674408 9"Purity/*FRt*SRqqt*FBt/event_purity‚0.2563784420490265 @&Purity/*FRt1SRqqt2FBt/event_proportion‚0.0006975446594879031 9"Purity/0FRt2SRqqt*FBt/event_purity‚0.3172382116317749 *jet/accuracy_1_of_1‚0.2804163098335266 *loss/val_total_loss‚10.313489241276079 & Purity/1FRt0SRqqt1FBt/FBt_purity‚1 @&Purity/1FRt1SRqqt2FBt/event_proportion‚0.0002325148816453293 8 Purity/2FRt*SRqqt0FBt/FRt_purity‚0.11400049179792404 9"Purity/0FRt*SRqqt1FBt/event_purity‚0.8498997688293457 7 Purity/0FRt*SRqqt1FBt/FBt_purity‚0.9959661364555359 7 Purity/1FRt*SRqqt2FBt/FRt_purity‚0.8859410881996155 ("Purity/1FRt2SRqqt*FBt/event_purity‚0 =&Purity/2FRt*SRqqt0FBt/event_proportion‚0.0382486991584301 ( Purity/2FRt1SRqqt2FBt/FRt_purity‚0.5 9"Purity/*FRt*SRqqt*FBt/SRqqt_purity‚0.6603255271911621 7"Purity/*FRt2SRqqt0FBt/SRqqt_purity‚0.64164799451828 7 Purity/0FRt*SRqqt2FBt/FBt_purity‚0.9840921759605408 9"Purity/0FRt2SRqqt*FBt/SRqqt_purity‚0.5600875020027161 @&Purity/1FRt*SRqqt2FBt/event_proportion‚0.0027204242069274187 7 Purity/1FRt2SRqqt0FBt/FRt_purity‚0.0594051294028759 /particle/accuracy_3_of_3‚0.9994933605194092 7 Purity/2FRt0SRqqt2FBt/FRt_purity‚0.8571428656578064 7 Purity/*FRt0SRqqt1FBt/FRt_purity‚0.6132236123085022 8"Purity/*FRt1SRqqt*FBt/SRqqt_purity‚0.667289137840271 9"Purity/*FRt2SRqqt*FBt/SRqqt_purity‚0.6362565159797668 7 Purity/*FRt2SRqqt*FBt/FBt_purity‚0.9814814329147339 =&Purity/1FRt0SRqqt*FBt/event_proportion‚0.4530552327632904 7 Purity/1FRt0SRqqt2FBt/FBt_purity‚0.9970238208770752 ("Purity/1FRt2SRqqt0FBt/event_purity‚0 8&Purity/1FRt2SRqqt0FBt/event_proportion‚ 0.02294921875 9"Purity/1FRt2SRqqt*FBt/SRqqt_purity‚0.6843730211257935 *jet/accuracy_1_of_3‚0.9112595915794373 *jet/accuracy_1_of_4‚0.9649112224578857 9"Purity/2FRt0SRqqt2FBt/event_purity‚0.7142857313156128 particle/accuracy_1_of_3‚1 trainer/global_step‚4399 A&Purity/2FRt0SRqqt2FBt/event_proportion‚0.00018601190822664648 <&Purity/*FRt*SRqqt0FBt/event_proportion‚0.881045401096344 8 Purity/*FRt1SRqqt1FBt/FRt_purity‚0.46795904636383057 =&Purity/0FRt*SRqqt0FBt/event_proportion‚0.2322126179933548 =&Purity/0FRt0SRqqt*FBt/event_proportion‚0.0390159972012043 9"Purity/1FRt*SRqqt0FBt/SRqqt_purity‚0.7306254506111145 ("Purity/1FRt1SRqqt2FBt/event_purity‚0 jet/accuracy_3_of_5‚1 7 Purity/*FRt2SRqqt0FBt/FRt_purity‚0.0662318766117096 ("Purity/0FRt0SRqqt0FBt/event_purity‚1 <"Purity/2FRt*SRqqt0FBt/event_purity‚0.0042312657460570335 =&Purity/2FRt1SRqqt*FBt/event_proportion‚0.0127883180975914 ?&Purity/2FRt2SRqqt*FBt/event_proportion‚0.006417410913854837 particle/accuracy_3_of_5‚1 <&Purity/*FRt2SRqqt*FBt/event_proportion‚0.054222472012043 9"Purity/1FRt1SRqqt2FBt/SRqqt_purity‚0.2222222238779068 & Purity/1FRt2SRqqt*FBt/FBt_purity‚1 ,&Purity/2FRt2SRqqt1FBt/event_proportion‚0 particle/accuracy_1_of_1‚1 =&Purity/1FRt1SRqqt*FBt/event_proportion‚0.1534598171710968 ("Purity/2FRt2SRqqt0FBt/event_purity‚0 +jet/accuracy_3_of_4‚0.07885383069515228 ("Purity/2FRt1SRqqt2FBt/SRqqt_purity‚0 ,&Purity/*FRt2SRqqt2FBt/event_proportion‚0 9"Purity/0FRt*SRqqt0FBt/SRqqt_purity‚0.6073150038719177 9"Purity/0FRt0SRqqt*FBt/event_purity‚0.9944429993629456 9"Purity/0FRt1SRqqt1FBt/event_purity‚0.5084317326545715 ,&Purity/0FRt2SRqqt2FBt/event_proportion‚0 7 Purity/1FRt0SRqqt*FBt/FRt_purity‚0.1312306672334671 :"Purity/1FRt1SRqqt0FBt/event_purity‚0.02052449807524681 & Purity/2FRt*SRqqt*FBt/FBt_purity‚1 9"Purity/*FRt1SRqqt1FBt/event_purity‚0.3963554799556732 8 Purity/1FRt1SRqqt1FBt/FRt_purity‚0.46628493070602417 7 Purity/2FRt2SRqqt*FBt/FRt_purity‚0.0762423649430275 ("Purity/2FRt1SRqqt2FBt/event_purity‚0 , Purity/0FRt2SRqqt*FBt/FBt_purity‚0.96875 ("Purity/1FRt2SRqqt1FBt/event_purity‚0 jet/accuracy_4_of_5‚0 7 Purity/*FRt*SRqqt1FBt/FRt_purity‚0.5455740094184875 :"Purity/*FRt1SRqqt2FBt/SRqqt_purity‚0.18518517911434174 8 Purity/*FRt2SRqqt*FBt/FRt_purity‚0.06568708270788193 & Purity/2FRt*SRqqt2FBt/FBt_purity‚1 epoch‚10 8"Purity/*FRt1SRqqt0FBt/SRqqt_purity‚0.672247052192688 & Purity/1FRt2SRqqt1FBt/FBt_purity‚1 ("Purity/2FRt2SRqqt*FBt/event_purity‚0 ("Purity/*FRt1SRqqt2FBt/event_purity‚0 @&Purity/*FRt2SRqqt1FBt/event_proportion‚0.0009998140158131719 7 Purity/1FRt*SRqqt1FBt/FRt_purity‚0.5533981919288635 8 Purity/2FRt1SRqqt1FBt/FRt_purity‚0.44675928354263306 particle/accuracy_2_of_3‚1 :"Purity/*FRt*SRqqt2FBt/SRqqt_purity‚0.18518517911434174 9"Purity/2FRt1SRqqt1FBt/SRqqt_purity‚0.7037036418914795 9"Purity/2FRt2SRqqt*FBt/SRqqt_purity‚0.7704119086265564 *jet/accuracy_1_of_2‚0.7681604027748108 7 Purity/2FRt0SRqqt1FBt/FRt_purity‚0.6041666269302368 7 Purity/*FRt0SRqqt2FBt/FRt_purity‚0.9052533507347107 8&Purity/*FRt2SRqqt0FBt/event_proportion‚ 0.05322265625 , Purity/0FRt2SRqqt1FBt/FBt_purity‚0.96875 ?&Purity/2FRt1SRqqt0FBt/event_proportion‚0.011928013525903225 @&Purity/2FRt1SRqqt1FBt/event_proportion‚0.0008370535797439516 9"Purity/*FRt*SRqqt2FBt/event_purity‚0.8559056520462036 9"Purity/0FRt2SRqqt0FBt/SRqqt_purity‚0.5658428072929382 8"Purity/0FRt*SRqqt1FBt/SRqqt_purity‚0.531871497631073 8 Purity/1FRt0SRqqt0FBt/FRt_purity‚0.11614324897527695 @&Purity/1FRt0SRqqt2FBt/event_proportion‚0.0024879092816263437 & Purity/2FRt*SRqqt1FBt/FBt_purity‚1 ?&Purity/2FRt2SRqqt0FBt/event_proportion‚0.006417410913854837 7 Purity/2FRt2SRqqt0FBt/FRt_purity‚0.0762423649430275 =&Purity/*FRt*SRqqt1FBt/event_proportion‚0.0617094486951828 7 Purity/*FRt*SRqqt2FBt/FBt_purity‚0.9841797351837158 9"Purity/0FRt1SRqqt0FBt/event_purity‚0.6171101331710815 7 Purity/2FRt*SRqqt1FBt/FRt_purity‚0.5216667056083679 6validation_average_jet_accuracy‚0.3298184871673584 7 Purity/0FRt1SRqqt1FBt/FBt_purity‚0.9902781248092651 >&Purity/0FRt2SRqqt*FBt/event_proportion‚0.02457682229578495 0"Purity/0FRt2SRqqt1FBt/SRqqt_purity‚ 0.3671875 9"Purity/1FRt0SRqqt*FBt/event_purity‚0.1291322410106659 +jet/accuracy_2_of_3‚0.48350587487220764 *jet/accuracy_2_of_4‚0.7162234783172607 :"Purity/1FRt*SRqqt1FBt/event_purity‚0.43137869238853455 _step‚362 7 Purity/1FRt*SRqqt2FBt/FBt_purity‚0.9854875206947327 9"Purity/1FRt0SRqqt1FBt/event_purity‚0.5602481961250305 7 Purity/1FRt1SRqqt*FBt/FBt_purity‚0.9906229376792908 @&Purity/2FRt0SRqqt1FBt/event_proportion‚0.0007672990905120969 7"Purity/2FRt1SRqqt0FBt/SRqqt_purity‚0.75511234998703 @&Purity/2FRt1SRqqt2FBt/event_proportion‚2.325148852833081e-05 & Purity/2FRt0SRqqt1FBt/FBt_purity‚1 & Purity/2FRt1SRqqt2FBt/FBt_purity‚1 7 Purity/*FRt1SRqqt*FBt/FBt_purity‚0.9860484600067139 & Purity/1FRt*SRqqt1FBt/FBt_purity‚1 & Purity/2FRt1SRqqt1FBt/FBt_purity‚1 & Purity/2FRt0SRqqt2FBt/FBt_purity‚1 jet/accuracy_1_of_5‚1 particle/accuracy_4_of_5‚1 particle/accuracy_5_of_5‚0 9"Purity/*FRt*SRqqt0FBt/SRqqt_purity‚0.6652052998542786 9"Purity/*FRt*SRqqt1FBt/SRqqt_purity‚0.5676465034484863 7 Purity/1FRt0SRqqt2FBt/FRt_purity‚0.9015306830406189 =&Purity/*FRt0SRqqt1FBt/event_proportion‚0.0418526791036129 9"Purity/0FRt0SRqqt1FBt/event_purity‚0.9984753131866455 & Purity/1FRt2SRqqt1FBt/FRt_purity‚0 +jet/accuracy_3_of_3‚0.06548494100570679 +particle/specificity‚0.9098367094993591 7 Purity/0FRt1SRqqt2FBt/FBt_purity‚0.9230769276618958 7 Purity/*FRt*SRqqt1FBt/FBt_purity‚0.9970881938934326 8 Purity/*FRt0SRqqt0FBt/FRt_purity‚0.11580312997102737 =&Purity/*FRt1SRqqt1FBt/event_proportion‚0.0188569575548172 9"Purity/0FRt1SRqqt*FBt/SRqqt_purity‚0.6127545833587646 >&Purity/1FRt*SRqqt1FBt/event_proportion‚0.01643880270421505 ;"Purity/1FRt1SRqqt*FBt/event_purity‚0.030215734615921974 9"Purity/1FRt2SRqqt0FBt/SRqqt_purity‚0.6891425251960754 ,&Purity/1FRt2SRqqt2FBt/event_proportion‚0 +jet/accuracy_2_of_2‚0.12510187923908234 7 Purity/*FRt1SRqqt2FBt/FBt_purity‚0.8796296119689941 9"Purity/*FRt2SRqqt0FBt/event_purity‚0.1454188972711563 7"Purity/0FRt0SRqqt2FBt/event_purity‚0.97723388671875 :"Purity/1FRt*SRqqt*FBt/event_purity‚0.10022103041410446 =&Purity/1FRt*SRqqt0FBt/event_proportion‚0.6105840802192688 & Purity/1FRt1SRqqt1FBt/FBt_purity‚1 A&Purity/1FRt2SRqqt1FBt/event_proportion‚0.00027901786961592734 & Purity/2FRt0SRqqt*FBt/FBt_purity‚1 =&Purity/*FRt0SRqqt*FBt/event_proportion‚0.5129278302192688 9"Purity/0FRt*SRqqt*FBt/SRqqt_purity‚0.6028573513031006 >&Purity/2FRt0SRqqt0FBt/event_proportion‚0.01990327425301075 jet/accuracy_2_of_5‚1ê÷ŸK“–‚(:ˆ ™€óÒä’¥ gpu.0.gpu‚91.0 gpu.0.memory‚19, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚47.0 gpu.0.powerWatts‚215.361' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚71.787 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ºs|éq—‚(:æ ™€óÒÄ°¦) proc.memory.percent‚17.87426471710205 cpu‚25.90527531919174 disk.dm-1.in‚0' disk.nvme0c0n1.out‚34102.6865234375$ memory_percent‚99.36020374298096# proc.memory.rssMB‚ 2928.51953125 proc.cpu.threads‚17) disk./.usagePercent‚76.33469331111141$ disk./.usageGB‚1171.928379058838 disk.md0.in‚0 disk.md0.out‚ 75.1796875" disk.dm-1.out‚1854.04833984375 disk.rbd0.in‚0 disk.rbd0.out‚0 network.recv‚816048$ disk.nvme1n1.out‚15929.212890625# disk.dm-0.out‚48015.95166015625" disk.nvme0c0n1.in‚ 1852.4453125 disk.nvme0n1.in‚ 1852.4453125& disk.nvme1c1n1.out‚15929.212890625 network.sent‚ 6.544808e+06 disk.nvme1n1.in‚ 129.765625 disk.dm-0.in‚ 1982.2109375% disk.nvme0n1.out‚34102.6865234375 disk.nvme1c1n1.in‚ 129.765625( proc.memory.availableMB‚ 104.82421875¸6m˜‚(:â ¨€óÒ¢ýݤ# disk.nvme1c1n1.out‚ 15929.984375 disk.md0.out‚ 75.49609375& proc.memory.availableMB‚ 104.765625 network.recv‚820763% disk./.usageGB‚1171.9283866882324! disk.dm-1.out‚1854.3251953125 disk.nvme1c1n1.in‚ 129.765625 disk.md0.in‚0 network.sent‚ 6.590983e+06 disk.dm-0.in‚ 1982.2109375 disk.dm-1.in‚0 disk.rbd0.out‚0 disk.nvme0n1.in‚ 1852.4453125! disk.nvme1n1.out‚ 15929.984375$ memory_percent‚99.36056137084961 proc.cpu.threads‚17# disk.dm-0.out‚48016.65478515625( disk.nvme0c0n1.out‚34103.58740234375& disk.nvme0n1.out‚34103.58740234375 disk.nvme1n1.in‚ 129.765625# proc.memory.rssMB‚ 2928.51953125) proc.memory.percent‚17.87426471710205 cpu‚25.903657048248167) disk./.usagePercent‚76.33469380805943 disk.rbd0.in‚0" disk.nvme0c0n1.in‚ 1852.4453125˜Û Ÿ™‚(:” ¨€óÒýž–¥ gpu.0.gpu‚86.0 gpu.0.memory‚21, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚47.0 gpu.0.powerWatts‚113.859' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚37.952999999999996 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ÿÿÉ2š- _wandbruntime‚3088  _runtime‚3088¶KÐyÇ›Á trainer/global_step‚4449 % _timestamp‚1.7844634141057508e+09 _runtime‚3088.494149789 _step‚363 & lr-AdamW/pg1‚0.0001250879827817557 & lr-AdamW/pg2‚0.0001250879827817557pÈwÙÌœÆ _runtime‚3088.494149789 _step‚363 & lr-AdamW/pg1‚0.0001250879827817557 & lr-AdamW/pg2‚0.0001250879827817557 trainer/global_step‚4449 % _timestamp‚1.7844634141057508e+09ëÁ•Ó2- _wandbruntime‚3088  _runtime‚3088½•äPóžÛO WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚345396544506368 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚33779307 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 $ _timestamp‚1.784463414106774e+09 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚198 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚35292821 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚344981855266501 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚23691306 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3672618127872 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚341319396013424 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 _runtime‚3088.495047971 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚35292823 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 219359232 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3672529467904 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚194 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚258695164945920 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3781178950144 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3662459253077 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3672618127872 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚349069073974272 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚44778674 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 _step‚364 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚35292823 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚345396544506368 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚349069179673600 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚80061352 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚258695259055616 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 125249536 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚80071497 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 trainer/global_step‚4449 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚10087807 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚44768531 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚23691302 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3781304199680 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3662546039503 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚349069073974272 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚262476563255296 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚345396561545728 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚10088001 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552赶5æ'ŸàO trainer/global_step‚4449 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚10087807 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚44768531 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚23691302 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3781304199680 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3662546039503 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚349069073974272 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚262476563255296 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚345396561545728 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚10088001 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚345396544506368 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚33779307 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 $ _timestamp‚1.784463414106774e+09 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚198 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚35292821 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚344981855266501 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚23691306 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3672618127872 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚341319396013424 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 _runtime‚3088.495047971 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚35292823 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 219359232 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3672529467904 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚194 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚258695164945920 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3781178950144 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3662459253077 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3672618127872 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚349069073974272 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚44778674 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 _step‚364 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚35292823 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚345396544506368 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚349069179673600 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚80061352 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚258695259055616 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 125249536 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚80071497 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130ì@Whß2 - _wandbruntime‚3088  _runtime‚3088‘œB¦ì&¡æM E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3673315183616 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚35300450 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚44778351 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚23696362 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚44788508 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 _step‚365 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚345471694172672 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚345471694172672 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚345471711212032 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚258751396783616 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚349145009356288 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3781991411712 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 118950912 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚173 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚258751302673920 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3663329534215 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚80088960 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚80088960 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚262533294085632 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3782110362624 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚341393686538816 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3673315183616 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚35300452 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚33786513 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚23696358 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 trainer/global_step‚4449 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚345471711212032 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚35300450 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3673403850752 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 213060608 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3673403850752 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚345056929286493 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚35300452 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚44778351 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚80078801 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚10090151 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚262533507146240 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚33786340 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚345057033112391 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0$ˆ¬Aë¢ëM YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 F6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚262533294085632 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3782110362624 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚341393686538816 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3673315183616 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚35300452 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚33786513 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚23696358 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 trainer/global_step‚4449 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚345471711212032 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚35300450 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3673403850752 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 213060608 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3673403850752 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚345056929286493 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚35300452 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚44778351 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚80078801 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚10090151 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚262533507146240 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚33786340 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚345057033112391 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3673315183616 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚35300450 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.currentW-„Ö ‚ 272629760 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚44778351 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚23696362 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚44788508 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 _step‚365 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚345471694172672 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚345471694172672 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚345471711212032 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚258751396783616 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚349145009356288 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3781991411712 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 118950912 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚173 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚258751302673920 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3663329534215 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚80088960 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚80088960 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696í4 ñ•2£- _wandbruntime‚3088  _runtime‚3088<,¤‚ /loss/FBt1/detection_loss‚0.5282354046080818 epoch‚11 _runtime‚3088.905983038 0loss/FRt1/assignment_loss‚2.5064951863649556 0loss/FRt2/assignment_loss‚1.0848181860879251 2loss/SRqqt1/assignment_loss‚0.6267682597007057 1loss/SRqqt1/detection_loss‚1.1471872712784577 1loss/FBt1/assignment_loss‚0.06304322750088583 1loss/SRqqt2/detection_loss‚0.6359219871275369 trainer/global_step‚4449 _step‚366 .loss/FRt2/detection_loss‚1.414976696538968 /loss/FBt2/detection_loss‚1.4893551703010286 &loss/total_loss‚10.840059314621392 % _timestamp‚1.7844634145179377e+09 2loss/SRqqt2/assignment_loss‚0.6364162107352418 1loss/FBt2/assignment_loss‚0.11309574367667535 /loss/FRt1/detection_loss‚0.5937459707009302’§’ ¥‡ 1loss/SRqqt2/detection_loss‚0.6359219871275369 trainer/global_step‚4449 _step‚366 .loss/FRt2/detection_loss‚1.414976696538968 /loss/FBt2/detection_loss‚1.4893551703010286 &loss/total_loss‚10.840059314621392 % _timestamp‚1.7844634145179377e+09 2loss/SRqqt2/assignment_loss‚0.6364162107352418 1loss/FBt2/assignment_loss‚0.11309574367667535 /loss/FRt1/detection_loss‚0.5937459707009302 /loss/FBt1/detection_loss‚0.5282354046080818 epoch‚11 _runtime‚3088.905983038 0loss/FRt1/assignment_loss‚2.5064951863649556 0loss/FRt2/assignment_loss‚1.0848181860879251 2loss/SRqqt1/assignment_loss‚0.6267682597007057 1loss/SRqqt1/detection_loss‚1.1471872712784577 1loss/FBt1/assignment_loss‚0.06304322750088583îLªTGr¦‚(:ç ·€óÒÙä ¥# proc.memory.rssMB‚ 2928.51953125) proc.memory.percent‚17.87426471710205 disk.nvme1n1.in‚ 129.765625% disk.nvme1n1.out‚15930.4501953125 disk.md0.out‚ 75.74609375" disk.dm-0.out‚48018.8779296875$ memory_percent‚99.36251640319824 cpu‚25.90209967276641 proc.cpu.threads‚17 network.sent‚ 6.606785e+06 disk.nvme1c1n1.in‚ 129.765625 disk.rbd0.in‚0) disk./.usagePercent‚76.33470250464984% disk./.usageGB‚1171.9285202026367% disk.nvme0n1.out‚34105.9091796875 disk.dm-0.in‚ 1982.2109375! disk.dm-1.out‚1854.3408203125 disk.rbd0.out‚0" disk.nvme0c0n1.in‚ 1852.4453125' disk.nvme0c0n1.out‚34105.9091796875 network.recv‚824191 disk.nvme0n1.in‚ 1852.4453125' disk.nvme1c1n1.out‚15930.4501953125 disk.md0.in‚0 disk.dm-1.in‚0' proc.memory.availableMB‚ 104.4453125tæG²’§‚(:‡ ·€óÒŵϥ gpu.0.gpu‚51.0 gpu.0.memory‚8, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚48.0 gpu.0.powerWatts‚222.606' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚74.202 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0€\Üa¨‚(:Ö Æ€óÒáû÷¤$ memory_percent‚99.36096668243408# proc.memory.rssMB‚ 2928.51953125) proc.memory.percent‚17.87426471710205 network.sent‚ 6.658016e+06% disk./.usageGB‚1171.9284934997559 disk.dm-1.in‚0 disk.rbd0.out‚0 disk.nvme1c1n1.in‚ 129.765625 disk.md0.in‚0 disk.dm-0.in‚ 1982.2109375( proc.memory.availableMB‚ 104.69921875 network.recv‚827741! disk.dm-1.out‚1854.4990234375" disk.nvme0c0n1.in‚ 1852.4453125 disk.nvme0n1.in‚ 1852.4453125 disk.nvme1n1.in‚ 129.765625" disk.dm-0.out‚48019.5380859375 proc.cpu.threads‚17 disk.rbd0.in‚0 disk.nvme0n1.out‚ 34106.96875! disk.nvme1n1.out‚ 15931.265625 disk.md0.out‚ 76.2421875 cpu‚25.900491801760154) disk./.usagePercent‚76.33470076533176" disk.nvme0c0n1.out‚ 34106.96875# disk.nvme1c1n1.out‚ 15931.265625î Qœ©‚(:‘ Æ€óÒËš©¥ gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚48.0 gpu.0.powerWatts‚223.147' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚74.38233333333332 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0øíº£sª‚(:è Õ€óÒØÝʤ network.sent‚ 6.660192e+06) disk./.usagePercent‚76.33475070860813% disk./.usageGB‚1171.9292602539062% disk.nvme1n1.out‚15933.8525390625 disk.md0.out‚ 76.6015625 disk.dm-1.in‚0 disk.rbd0.in‚0 disk.rbd0.out‚0 disk.nvme0n1.in‚ 1852.4453125' disk.nvme1c1n1.out‚15933.8525390625! disk.dm-1.out‚1856.2607421875" disk.nvme0c0n1.in‚ 1852.4453125$ memory_percent‚99.36096668243408( proc.memory.availableMB‚ 104.69921875# proc.memory.rssMB‚ 2928.51953125 cpu‚25.898979672980047 disk.nvme1n1.in‚ 129.765625 disk.md0.in‚0" disk.dm-0.out‚48021.7275390625) proc.memory.percent‚17.87426471710205 proc.cpu.threads‚17 network.recv‚831108% disk.nvme0n1.out‚34109.0927734375 disk.nvme1c1n1.in‚ 129.765625 disk.dm-0.in‚ 1982.2109375' disk.nvme0c0n1.out‚34109.0927734375² ÔÁ’«‚(:‡ Õ€óÒÛΊ¥ gpu.0.gpu‚47.0 gpu.0.memory‚8, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚48.0 gpu.0.powerWatts‚217.557' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚72.519 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0'DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚265323811206144 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚352866012929024 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3711968068096 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚45271526 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 trainer/global_step‚4499 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚348735620066909 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 _runtime‚3120.752623488 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚35674271 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚45271526 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚10196508 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚349154044860928 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 % _timestamp‚1.7844634463643892e+09 _step‚368 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚352865907229696 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚34140445 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚352865907229696 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚349154027821568 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚35674273 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚35674271 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚45261383 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3711968068096 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚165 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚261502145455616 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚348735723892695 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚34140614 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚23944102 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚349154027821568 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚80945799 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚35674273 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚10196343 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚261502051345920 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚349154044860928 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 118958080 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚345033922283024 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 ráXç'±áO PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚35674271 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚45261383 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3711968068096 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚165 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚261502145455616 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚348735723892695 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚34140614 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚23944102 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚349154027821568 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚80945799 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚35674273 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚10196343 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚261502051345920 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚349154044860928 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 118958080 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚345033922283024 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚45261383 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚23944106 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3821878818304 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚265323811206144 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚352866012929024 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3711968068096 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚45271526 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 trainer/global_step‚4499 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚348735620066909 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 _runtime‚3120.752623488 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚35674271 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚45271526 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚10196508 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚349154044860928 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 % _timestamp‚1.7844634463643892e+09 _step‚368 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚352865907229696 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚34140445 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚352865907229696 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚349154027821568 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚35674273 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736ð|hE.2²- _wandbruntime‚3121  _runtime‚3121ñËŽ9 ³æM ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚34147664 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚23949162 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3712753790976 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚261558189073920 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 trainer/global_step‚4499 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3712665123840 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚45281360 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚352941842611712 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 219352064 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 150006272 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 _step‚369 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚349229194527232 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 _runtime‚3121.159365624 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚349229177487872 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚345108229847776 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3702568065023 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚80963262 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 125242368 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚35681902 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚35681902 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚352941948318208 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚265380972121600 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚348810797912799 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚45271203 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3712665123840 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚45271203 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚35681900 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚45281360 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚352941948318208 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚349229194527232 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚191 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚80953103 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3712753790976 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚348810694086901 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3702481278485 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚10198506 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚265380752769536 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 % _timestamp‚1.7844634467708573e+09 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚80963262 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚345108212808416 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚349229177487872 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚261558283183616( Š5ñ&´ëM UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3712665123840 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚45271203 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚35681900 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚45281360 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚352941948318208 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚349229194527232 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚191 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚80953103 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3712753790976 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚348810694086901 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3702481278485 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚10198506 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚265380752769536 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 % _timestamp‚1.7844634467708573e+09 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚80963262 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚345108212808416 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚349229177487872 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚261558283183616 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚34147664 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚23949162 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3712753790976 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚261558189073920 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 trainer/global_step‚4499 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3712665123840 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚45281360 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚352941842611712 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 219352064 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 150006272 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 _step‚369 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚349229194527232 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 _runtime‚3121.159365624 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚349229177487872 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚345108229847776 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3702568065023 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚80963262 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 125242368 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚35681902 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚35681902 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚352941948318208 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚265380972121600 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚348810797912799 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚45271203ñ‡©/42µ- _wandbruntime‚3121  _runtime‚3121D“* ¶ƒ trainer/global_step‚4499 % _timestamp‚1.7844634467718112e+09 0loss/FRt1/assignment_loss‚2.5830321841198822 2loss/SRqqt2/assignment_loss‚0.5926103764171862 1loss/FBt2/assignment_loss‚0.15176037834810152 /loss/FRt1/detection_loss‚0.5648811040184291 /loss/FBt1/detection_loss‚0.5558099933259019 /loss/FBt2/detection_loss‚1.0320210034648578 2loss/SRqqt1/assignment_loss‚0.6604583841851461 1loss/SRqqt2/detection_loss‚0.7672838061868518 _step‚370 1loss/FBt1/assignment_loss‚0.07458127191906558 /loss/FRt2/detection_loss‚1.4490592425067617 1loss/SRqqt1/detection_loss‚1.1077152431638744 &loss/total_loss‚10.573911460300629 _runtime‚3121.159913494 0loss/FRt2/assignment_loss‚1.0346984726445685 epoch‚11/ë]i·ˆ epoch‚11 trainer/global_step‚4499 % _timestamp‚1.7844634467718112e+09 0loss/FRt1/assignment_loss‚2.5830321841198822 2loss/SRqqt2/assignment_loss‚0.5926103764171862 1loss/FBt2/assignment_loss‚0.15176037834810152 /loss/FRt1/detection_loss‚0.5648811040184291 /loss/FBt1/detection_loss‚0.5558099933259019 /loss/FBt2/detection_loss‚1.0320210034648578 2loss/SRqqt1/assignment_loss‚0.6604583841851461 1loss/SRqqt2/detection_loss‚0.7672838061868518 _step‚370 1loss/FBt1/assignment_loss‚0.07458127191906558 /loss/FRt2/detection_loss‚1.4490592425067617 1loss/SRqqt1/detection_loss‚1.1077152431638744 &loss/total_loss‚10.573911460300629 _runtime‚3121.159913494 0loss/FRt2/assignment_loss‚1.0346984726445685ò%bÐìn¸‚(:ã ä€óÒÍøø¤ disk.nvme1c1n1.in‚ 129.765625) proc.memory.percent‚17.87426471710205 network.sent‚ 6.677886e+06% disk.nvme1n1.out‚15934.7744140625 disk.md0.out‚ 77.0703125 disk.dm-0.in‚ 1982.2109375& disk.nvme0c0n1.out‚34110.216796875) disk./.usagePercent‚76.31926804446103 disk.nvme1n1.in‚ 129.765625 disk.dm-1.in‚0 disk.rbd0.in‚0 disk.nvme0n1.in‚ 1852.4453125 cpu‚25.900089933773675 network.recv‚834864 disk.dm-1.out‚ 1856.44921875 disk.rbd0.out‚0' disk.nvme1c1n1.out‚15934.7744140625$ memory_percent‚99.36120510101318( proc.memory.availableMB‚ 104.66015625# proc.memory.rssMB‚ 2928.51953125 proc.cpu.threads‚17$ disk./.usageGB‚1171.691562652588 disk.md0.in‚0" disk.dm-0.out‚48022.5634765625" disk.nvme0c0n1.in‚ 1852.4453125$ disk.nvme0n1.out‚34110.216796875£Q(ž¹‚(:“ ä€óҕ毥 gpu.0.gpu‚91.0 gpu.0.memory‚17, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚48.0 gpu.0.powerWatts‚213.551' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚71.18366666666667 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0H‘º‚(:† ó€óÒ¤Žà¤ gpu.0.gpu‚2.0 gpu.0.memory‚1, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚47.0 gpu.0.powerWatts‚113.514' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚37.838 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0+¡ýÓf»‚(:Û ó€óÒþè±¥ network.recv‚838618 disk.nvme1c1n1.in‚ 129.765625# disk.nvme1n1.out‚15937.30859375 disk.rbd0.in‚0' disk.nvme0c0n1.out‚34121.4208984375 disk.nvme0n1.in‚ 1852.4453125) proc.memory.percent‚17.87426471710205% disk./.usageGB‚1171.6912651062012 disk.dm-0.in‚ 1982.2109375 disk.rbd0.out‚0" disk.nvme0c0n1.in‚ 1852.4453125% disk.nvme0n1.out‚34121.4208984375$ memory_percent‚99.36378002166748 cpu‚25.90126730513253 proc.cpu.threads‚17) disk./.usagePercent‚76.31924866348811% disk.nvme1c1n1.out‚15937.30859375 disk.nvme1n1.in‚ 129.765625 disk.dm-1.in‚0 network.sent‚ 6.729304e+06 disk.md0.in‚0 disk.md0.out‚ 77.52734375" disk.dm-0.out‚48035.2763671875 disk.dm-1.out‚1856.5( proc.memory.availableMB‚ 104.23828125# proc.memory.rssMB‚ 2928.51953125µìö2¼- _wandbruntime‚3153  _runtime‚3153«#XlǽÁ _step‚371 & lr-AdamW/pg1‚0.0001244064355554196 & lr-AdamW/pg2‚0.0001244064355554196 trainer/global_step‚4549 % _timestamp‚1.7844634794636252e+09 _runtime‚3153.852392189¬ÞHÌ¾Æ & lr-AdamW/pg1‚0.0001244064355554196 & lr-AdamW/pg2‚0.0001244064355554196 trainer/global_step‚4549 % _timestamp‚1.7844634794636252e+09 _runtime‚3153.852392189 _step‚371óÁ›oR2¿- _wandbruntime‚3153  _runtime‚3153zó÷á'ÀÛO TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚36055723 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚45764378 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚24196902 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 129443840 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚36055721 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚24196906 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚10304339 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚352911511136768 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚268171247369216 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚45754235 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚36055721 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3740936314693 _step‚372 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3751318008320 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚348748465591984 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚352911528176128 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚356662846184448 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚352489488693103 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚348748448552624 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚356662846184448 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3862309623296 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚81809956 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚171 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 _runtime‚3153.852598339 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚175 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 223553536 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3862439067136 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚45764378 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚34501070 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚356662740485120 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚34501245 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚36055723 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143=y¤‡ÏÁàO D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚36055723 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚36055723 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 PCDeviceStatsM)n%onitor.on_train_batch_start/active.small_pool.allocated‚45764378 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚24196902 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 129443840 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚36055721 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚24196906 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚10304339 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚352911511136768 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚268171247369216 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚45754235 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚36055721 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3740936314693 _step‚372 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3751318008320 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚348748465591984 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚352911528176128 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚356662846184448 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚352489488693103 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚348748448552624 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚356662846184448 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3862309623296 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚81809956 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚171 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 _runtime‚3153.852598339 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚175 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 223553536 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3862439067136 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚45764378 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚34501070 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚356662740485120 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚34501245 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145ô‚EÞ.2Â- _wandbruntime‚3154  _runtime‚3154ò­Ëì&ÃæM K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚81837564 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚36063352 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚10306519 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3741806595831 % _timestamp‚1.7844634798760638e+09 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚24201958 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚348822756117376 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3752015064064 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 150006272 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚264365075473920 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚352986677842432 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚352986660803072 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚356738781573632 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚356738781573632 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚81827405 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚36063352 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚352986660803072 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚24201962 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚45764055 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚348822739078016 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚10306341 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚34508299 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚264365169583616 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 _runtime‚3154.264551694 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚182 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 221449216 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚81837564 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚34508481 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3741719809293 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 _step‚373 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚268228415624704 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 trainer/global_step‚4549:ðTGñ&ÄëM UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚45774212 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 127339520 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3863118701568 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚34508481 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3741719809293 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 _step‚373 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚268228415624704 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 trainer/global_step‚4549 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚81837564 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚36063352 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚10306519 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3741806595831 % _timestamp‚1.7844634798760638e+09 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚24201958 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚348822756117376 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3752015064064 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 150006272 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚264365075473920 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚352986677842432 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚352986660803072 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚356738781573632 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚356738781573632 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚81827405 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚36063352 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚352986660803072 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚24201962 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚45764055 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚348822739078016 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚10306341 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚34508299 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚264365169583616 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 _runtime‚3154.264551694 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚182 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 221449216 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚81837564 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 FDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚360459573740544 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚356669011491328 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚24449702 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚36437171 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚46247087 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚356243149667725 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚360459679439872 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 150006272 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚24449706 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚356668994451968 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚352462974822224 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚34860356 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚267115918255616 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚356668994451968 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚82684258 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 % _timestamp‚1.7844635121554568e+09 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3790667948544 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚34860183 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚356243253493511 _runtime‚3186.543969415 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚356669011491328 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚271018794127872 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚267115824145920 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚46247087 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚169 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3902746428416 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚82694403 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚360459573740544 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 223553536 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 trainer/global_step‚4599ºSÖ©ç'ÑáO RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚360459573740544 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚356669011491328 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚24449702 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚36437171 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚46247087 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚356243149667725 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚360459679439872 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 150006272 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚24449706 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚356668994451968 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚352462974822224 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚34860356 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚267115918255616 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚356668994451968 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚82684258 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 % _timestamp‚1.7844635121554568e+09 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3790667948544 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚34860183 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚356243253493511 _runtime‚3186.543969415 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚356669011491328 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚271018794127872 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚267115824145920 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚46247087 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚169 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3902746428416 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚82694403 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚360459573740544 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 223553536 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 trainer/global_step‚4599 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚46257230 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚173 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 129443840 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 _step‚376 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3780261631927 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚360459679439872 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3790579288576 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚10410481 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚46257230 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3790579288576 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚36437173 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚10410650 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3780174845501 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚36437171 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚271018570574336 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚36437173 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682øA~Þ2Ò- _wandbruntime‚3186  _runtime‚3186£Óì&ÓæM TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3903558128128 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚356318223687717 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3903687564800 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 trainer/global_step‚4599 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚82711866 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚178 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3780958340101 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚36444800 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚356744161157632 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚267172055983616 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚271075743548416 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚24454762 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚36444800 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚46267064 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚356744144118272 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚352537265347616 _step‚377 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚34867324 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚356744144118272 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚82711866 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚356318327513615 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3781045126639 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚10412566 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚360535509122560 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚36444802 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚46267064 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚360535614829056 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚36444802 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3791365004288 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3791453671424 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚360535614829056 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚24454758 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 % _timestamp‚1.7844635125781658e+09 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚82701707 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚352537282386976 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚46256907 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 223546368 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 129436672 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 150006272 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3903558128128 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚356318223687717 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3903687564800 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 trainer/global_step‚4599 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚82711866 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚178 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3780958340101 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚36444800 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚356744161157632 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚267172055983616 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚271075743548416 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚24454762 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚36444800 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚46267064 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚356744144118272 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚352537265347616 _step‚377 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚34867324 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚356744144118272 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚82711866 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚356318327513615 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3781045126639 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚10412566 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚360535509122560 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚36444802 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚46267064 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚360535614829056 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚36444802 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3791365004288 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3791453671424 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚360535614829056 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚24454758 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 % _timestamp‚1.7844635125781658e+09 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚82701707 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚352537282386976 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚46256907 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 223546368 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 129436672 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 150006272 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚273866161496064 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚360426477767168 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3819413376309 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚10517563 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚359997018293919 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 225650688 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚356177518131184 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚46750082 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚46739939 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚364256512695296 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚364256512695296 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 % _timestamp‚1.7844635442081044e+09 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚360426477767168 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚36818623 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚178 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚24702506 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3943356840448 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚83568705 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚36818621 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚35219887 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 _runtime‚3218.599804505 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚269922710545920 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚364256406995968 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚46750082 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚356177501091824 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚83558560 trainer/global_step‚4649 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚182 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960ØBë%‘!ááO M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚83568705 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚36818621 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚35219887 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 _runtime‚3218.599804505 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚269922710545920 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚364256406995968 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚46750082 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚356177501091824 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚83558560 trainer/global_step‚4649 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚182 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚364256406995968 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 131540992 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 150006272 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚10517385 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚273866161496064 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚360426477767168 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3819413376309 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚10517563 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚359997018293919 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 225650688 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚356177518131184 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚46750082 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚46739939 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚364256512695296 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚364256512695296 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 % _timestamp‚1.7844635442081044e+09 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚360426477767168 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚36818623 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚178 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚24702506 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3943356840448 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200ü_XûS2â- _wandbruntime‚3218  _runtime‚3218Ø®Ÿ ì&ãæM UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 _step‚381 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚4649 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚36826250 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚46759916 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚24707562 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚10519626 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚83576009 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚356251808656576 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3820196870909 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚83586168 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚175 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚35227188 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚360071988488125 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3830803611648 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚273923103445504 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚360072092314023 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚83586168 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚35227009 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚273922879899136 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3820283657447 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 _runtime‚3218.994659964 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3830803611648 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚10519451 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚83576009 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚46749759 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚360501627433472 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚179 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248ÂØñ&äëM OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚46759916 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚24707562 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚10519626 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚83576009 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚356251808656576 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3820196870909 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚83586168 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚175 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚35227188 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚360071988488125 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3830803611648 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚273923103445504 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚360072092314023 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚83586168 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚35227009 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚273922879899136 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3820283657447 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 _runtime‚3218.994659964 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3830803611648 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚10519451 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚83576009 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚46749759 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚360501627433472 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚179 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 _step‚381 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 trainer/global_step‚4649 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚36826250ý¤™‘I2å- _wandbruntime‚3218  _runtime‚3218z6¶ æƒ /loss/FBt1/detection_loss‚0.5188388274699295 &loss/total_loss‚10.227376922127956 0loss/FRt1/assignment_loss‚2.4313270779807916 0loss/SRqqt1/detection_loss‚1.069786206460053 1loss/SRqqt2/detection_loss‚0.7202230239401496 epoch‚11 $ _timestamp‚1.784463544607098e+09 0loss/FRt2/assignment_loss‚1.1369978101713936 2loss/SRqqt1/assignment_loss‚0.5985956289973783 2loss/SRqqt2/assignment_loss‚0.6189145901001616 2loss/FBt2/assignment_loss‚0.048249013920498345 /loss/FBt2/detection_loss‚0.9952014548083147 trainer/global_step‚4649 _runtime‚3218.995151284 /loss/FRt2/detection_loss‚1.4743692404465896 _step‚382 2loss/FBt1/assignment_loss‚0.027757751208200718 /loss/FRt1/detection_loss‚0.5871162966244954‡Jù©çˆ trainer/global_step‚4649 _runtime‚3218.995151284 /loss/FRt2/detection_loss‚1.4743692404465896 _step‚382 2loss/FBt1/assignment_loss‚0.027757751208200718 /loss/FRt1/detection_loss‚0.5871162966244954 /loss/FBt1/detection_loss‚0.5188388274699295 &loss/total_loss‚10.227376922127956 0loss/FRt1/assignment_loss‚2.4313270779807916 0loss/SRqqt1/detection_loss‚1.069786206460053 1loss/SRqqt2/detection_loss‚0.7202230239401496 epoch‚11 $ _timestamp‚1.784463544607098e+09 0loss/FRt2/assignment_loss‚1.1369978101713936 2loss/SRqqt1/assignment_loss‚0.5985956289973783 2loss/SRqqt2/assignment_loss‚0.6189145901001616 2loss/FBt2/assignment_loss‚0.048249013920498345 /loss/FBt2/detection_loss‚0.9952014548083147þï}|ëtè‚(:é ¾óÒã™ß¤& disk.nvme0n1.out‚34134.92919921875 disk.nvme1c1n1.in‚ 129.765625 disk.md0.in‚0 disk.dm-0.in‚ 1982.2109375 disk.rbd0.in‚0( proc.memory.availableMB‚ 104.82421875 network.sent‚ 6.866773e+06' disk.nvme1c1n1.out‚15940.6416015625 disk.rbd0.out‚0" disk.nvme0c0n1.in‚ 1852.4453125$ memory_percent‚99.36020374298096) proc.memory.percent‚17.87426471710205 cpu‚25.90202077215085( disk.nvme0c0n1.out‚34134.92919921875# proc.memory.rssMB‚ 2928.51953125 proc.cpu.threads‚17% disk./.usageGB‚1168.7370567321777 disk.nvme0n1.in‚ 1852.4453125 disk.nvme1n1.in‚ 129.765625% disk.nvme1n1.out‚15940.6416015625 disk.md0.out‚ 79.01953125# disk.dm-0.out‚48048.02587890625 disk.dm-1.in‚0 disk.dm-1.out‚ 1857.30859375 network.recv‚856602) disk./.usagePercent‚76.12682343150482é3Ôé‚(:’ ¾óÒªêÆ¥ gpu.0.gpu‚91.0 gpu.0.memory‚21, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚48.0 gpu.0.powerWatts‚119.35' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚39.78333333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0&µ+mê‚(:â Íóҭ鿤& disk.nvme0c0n1.out‚34151.990234375 disk.nvme1c1n1.in‚ 129.765625' disk.nvme1c1n1.out‚15950.2822265625 disk.md0.in‚0 disk.dm-1.out‚ 1864.921875 disk.rbd0.in‚0) proc.memory.percent‚17.87426471710205 cpu‚25.901170273814063 proc.cpu.threads‚17 network.recv‚860341 disk.nvme0n1.in‚ 1852.4453125 disk.dm-0.in‚ 1982.2109375" disk.dm-0.out‚48066.2392578125# memory_percent‚99.3617296218872 network.sent‚ 6.930363e+06$ disk.nvme0n1.out‚34151.990234375 disk.md0.out‚ 79.43359375 disk.dm-1.in‚0 disk.rbd0.out‚0( proc.memory.availableMB‚ 104.57421875# proc.memory.rssMB‚ 2928.51953125 disk.nvme1n1.in‚ 129.765625% disk.nvme1n1.out‚15950.2822265625) disk./.usagePercent‚76.12682169218674% disk./.usageGB‚1168.7370300292969" disk.nvme0c0n1.in‚ 1852.4453125±P8žë‚(:“ ÍóÒÑÂî¤ gpu.0.gpu‚32.0 gpu.0.memory‚6, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚47.0 gpu.0.powerWatts‚140.683' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚46.894333333333336 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0FOÛ2ì- _wandbruntime‚3251  _runtime‚3251ÙäJ5ÇíÁ & lr-AdamW/pg1‚0.0001233577802430226 & lr-AdamW/pg2‚0.0001233577802430226 trainer/global_step‚4699 % _timestamp‚1.7844635767317178e+09 _runtime‚3251.120098222 _step‚383øœDÌîÆ trainer/global_step‚4699 % _timestamp‚1.7844635767317178e+09 _runtime‚3251.120098222 _step‚383 & lr-AdamW/pg1‚0.0001233577802430226 & lr-AdamW/pg2‚0.0001233577802430226ÿ28BÚ2ï- _wandbruntime‚3251  _runtime‚3251pgCÏðÜO N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚364183978121728 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚363750679268541 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3869367828992 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3983910959616 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3983781515776 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚364183978121728 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3858738693543 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚24955306 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚276713378461696 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚84443007 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚272729596945920 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚37200073 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚35579492 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚272729691055616 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚47242934 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚368053345950720 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚24955302 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 % _timestamp‚1.7844635767326512e+09 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚47232791 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 M@DeviceStats%Ä)»Monitor.on_train_batch_start/inactive_split.all.freed‚35579311 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚37200071 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚368053345950720 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3869367828992 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚359892044400784 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚47242934 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 129443840 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚364183961082368 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚10624009 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚276713602015232 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 150006272 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚368053240251392 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426UXþç'ñáO H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚24955306 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚276713378461696 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚84443007 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚272729596945920 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚37200073 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚35579492 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚272729691055616 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚47242934 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚368053345950720 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚24955302 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 % _timestamp‚1.7844635767326512e+09 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚47232791 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚35579311 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚37200071 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚368053345950720 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3869367828992 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚359892044400784 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚47242934 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 129443840 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚364183961082368 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚10624009 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚276713602015232 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 150006272 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚368053240251392 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚364183978121728 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚363750679268541 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3869367828992 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 3983910959616 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 3983781515776 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚364183978121728 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3858738693543 IDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚84460470 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚359966334926176 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚368129281339904 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 217254912 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 trainer/global_step‚4699 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3984717016064 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚47242611 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚84460470 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚359966317886816 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚363825857114431 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚364259110748672 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 _runtime‚3251.530948412 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚47252768 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚37207702 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 150006272 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚47242611 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚10626365 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚173 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚24960358 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚363825753288533 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚37207700 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚37207702 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚37207700 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚35586554 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚276770328544768 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚276770545799680 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3859522188255 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3984593870848 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3870153551872 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3870064884736 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚35586727 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0@—iÅ!ôëM LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 3984593870848 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3870153551872 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3870064884736 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚35586727 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚84460470 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚359966334926176 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚368129281339904 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 217254912 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 trainer/global_step‚4699 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 3984717016064 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚47242611 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚84460470 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚359966317886816 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚363825857114431 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚364259110748672 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 _runtime‚3251.530948412 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚47252768 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚37207702 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 150006272 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚47242611 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚10626365 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚173 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚24960358 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚363825753288533 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚37207700 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚37207702 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚37207700 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚35586554 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚276770328544768 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚276770545799680 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 38595221882556ÑÛ&2õ- _wandbruntime‚3251  _runtime‚3251OKì öÿ /loss/FRt1/assignment_loss‚2.350390740947784 1loss/SRqqt1/assignment_loss‚0.648379807469435 /loss/FRt1/detection_loss‚0.5406994437553111 &loss/total_loss‚10.256186910545942 trainer/global_step‚4699 _step‚386 1loss/SRqqt2/assignment_loss‚0.627848912389166 /loss/FBt1/detection_loss‚0.5988156466841184 0loss/FRt2/assignment_loss‚1.1898586246037575 1loss/FBt2/assignment_loss‚0.08312334425843677 1loss/SRqqt2/detection_loss‚0.6890222177927362 /loss/FBt2/detection_loss‚0.9634994130868179 1loss/FBt1/assignment_loss‚0.06755343840206385 /loss/FRt2/detection_loss‚1.3973753326999483 1loss/SRqqt1/detection_loss‚1.0996199884563684 epoch‚11 $ _timestamp‚1.784463577143422e+09 _runtime‚3251.531464173œ…› ÷„ 1loss/SRqqt2/detection_loss‚0.6890222177927362 /loss/FBt2/detection_loss‚0.9634994130868179 1loss/FBt1/assignment_loss‚0.06755343840206385 /loss/FRt2/detection_loss‚1.3973753326999483 1loss/SRqqt1/detection_loss‚1.0996199884563684 epoch‚11 $ _timestamp‚1.784463577143422e+09 _runtime‚3251.531464173 /loss/FRt1/assignment_loss‚2.350390740947784 1loss/SRqqt1/assignment_loss‚0.648379807469435 /loss/FRt1/detection_loss‚0.5406994437553111 &loss/total_loss‚10.256186910545942 trainer/global_step‚4699 _step‚386 1loss/SRqqt2/assignment_loss‚0.627848912389166 /loss/FBt1/detection_loss‚0.5988156466841184 0loss/FRt2/assignment_loss‚1.1898586246037575 1loss/FBt2/assignment_loss‚0.08312334425843677‚è䶪tø‚(:é ÜóÒ¬¡Æ¤% disk./.usageGB‚1168.7652778625488 disk.dm-1.in‚0 disk.rbd0.out‚0& disk.nvme0c0n1.out‚34154.232421875 disk.nvme0n1.in‚ 1852.4453125& disk.nvme1n1.out‚15950.84228515625 cpu‚25.90261735449096) disk./.usagePercent‚76.12866164224386 disk.md0.in‚0 disk.md0.out‚ 79.75390625 disk.dm-0.in‚ 1982.2109375$ disk.nvme0n1.out‚34154.232421875# proc.memory.rssMB‚ 2928.51953125 proc.cpu.threads‚17" disk.dm-1.out‚1865.04833984375 disk.rbd0.in‚0" disk.nvme0c0n1.in‚ 1852.4453125 disk.nvme1n1.in‚ 129.765625$ memory_percent‚99.36182498931885) proc.memory.percent‚17.87426471710205 network.sent‚ 6.934086e+06 network.recv‚863605" disk.dm-0.out‚48068.2119140625 disk.nvme1c1n1.in‚ 129.765625( disk.nvme1c1n1.out‚15950.84228515625( proc.memory.availableMB‚ 104.558593753Í^‘ù‚(:† ÜóÒ°Ê­¥ gpu.0.gpu‚90.0 gpu.0.memory‚16, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚49.0 gpu.0.powerWatts‚212.73' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚70.91 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ΠÄžú‚(:“ ëóÒ¯ª¤ gpu.0.gpu‚80.0 gpu.0.memory‚19, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚49.0 gpu.0.powerWatts‚212.183' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚70.72766666666666 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0C”yû‚(:î ëóÒ¥•²¥# disk.dm-0.out‚48072.32080078125 disk.dm-1.in‚0# proc.memory.rssMB‚ 2928.51953125) disk./.usagePercent‚76.04955720738428& disk.nvme0n1.out‚34158.65185546875( disk.nvme1c1n1.out‚15951.66455078125 disk.nvme1n1.in‚ 129.765625& disk.nvme1n1.out‚15951.66455078125 disk.md0.out‚ 80.1484375 network.sent‚ 6.996475e+06 disk.dm-0.in‚ 1982.2109375$ memory_percent‚99.36215877532959) proc.memory.percent‚17.87426471710205 cpu‚25.901221026799963 proc.cpu.threads‚17( disk.nvme0c0n1.out‚34158.65185546875 disk.nvme1c1n1.in‚ 129.765625" disk.dm-1.out‚1865.33349609375 disk.rbd0.in‚0 disk.rbd0.out‚0( proc.memory.availableMB‚ 104.50390625 network.recv‚867667% disk./.usageGB‚1167.5508270263672" disk.nvme0c0n1.in‚ 1852.4453125 disk.nvme0n1.in‚ 1852.4453125 disk.md0.in‚0U»#2ü- _wandbruntime‚3282  _runtime‚3282íŽÉýà trainer/global_step‚4749 % _timestamp‚1.7844636081062682e+09 _runtime‚3282.495525137 _step‚387 ' lr-AdamW/pg1‚0.00012300129552568167 ' lr-AdamW/pg2‚0.00012300129552568167d\”íÎþÈ ' lr-AdamW/pg1‚0.00012300129552568167 ' lr-AdamW/pg2‚0.00012300129552568167 trainer/global_step‚4749 % _timestamp‚1.7844636081062682e+09 _runtime‚3282.495525137 _step‚387ƒ!"i2ÿ- _wandbruntime‚3282  _runtime‚3282號â'€ÜO KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚47725643 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3908717769216 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 223553536 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚367504444068949 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚190 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 _step‚388 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚371850073506816 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚37581523 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚10731855 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚275536483345920 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 150006272 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚371850179206144 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚367504547894735 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚25208102 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚371850073506816 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚367941444397568 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 % _timestamp‚1.7844636081069753e+09 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚85307164 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚37581523 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3897977224351 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚367941444397568 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3908629109248 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚47725643 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚37581521 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚35939957 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚363606570670384 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3908717769216 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚371850179206144 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚279560924178432 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚367941461436928 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4024346722816 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚37581521 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚275536577455616 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚371850179206144 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚367504547894735 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚25208102 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚371850073506816 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚367941444397568 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 % _timestamp‚1.7844636081069753e+09 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚85307164 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚37581523 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3897977224351 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚367941444397568 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3908629109248 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚47725643 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚37581521 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚35939957 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚363606570670384 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3908717769216 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚371850179206144 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚279560924178432 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚367941461436928 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4024346722816 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚37581521 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚275536577455616 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚371850073506816 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚37581523 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚10731855 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚275536483345920 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 150006272 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786„Ž W2‚- _wandbruntime‚3282  _runtime‚3282ÞOçƒæM VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚10734168 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚279617649790464 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 _step‚389 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚368016611103232 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚35947330 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3909503492096 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚35947144 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚85334772 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚371926008888832 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚371926114595328 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3909503492096 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚182 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚367579621914839 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚85334772 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚47735463 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3909414824960 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 150006272 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚85324613 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚85324613 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚368016594063872 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4025028716544 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚363680844156416 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚10733986 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚35947144 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚85334772 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚371926008888832 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚371926114595328 E8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 3909503492096 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚182 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚367579621914839 PDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚85334772 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚47735463 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3909414824960 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 150006272 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚85324613 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚85324613 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚368016594063872 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4025028716544 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚363680844156416 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚10733986 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 FDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3948067709440 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚367321079900624 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3947979049472 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚375646906762240 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚375647012461568 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚375646906762240 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚375647012461568 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚371698944752128 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚48218495 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 $ _timestamp‚1.784463638984882e+09 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚371698944752128 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚36299865 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚278343463855616 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4064830657024 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚367321096939984 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚10838786 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3947979049472 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚86191611 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 trainer/global_step‚4799 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚371698927712768 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚37962971 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚48228638 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3948067709440 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚25460906 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚177 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚36299688 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚282408070959104 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚371258208869357 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚25460902 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚371698927712768 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚173 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 129443840 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚371258312695143 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 150006272Ô*(Ù‘‘àO JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚177 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚36299688 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚282408070959104 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚371258208869357 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚25460902 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚371698927712768 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚173 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 129443840 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚371258312695143 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 150006272 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚10838959 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚37962971 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚48228638 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚278343369745920 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 3948067709440 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚367321079900624 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 3947979049472 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚375646906762240 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚375647012461568 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚375646906762240 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚375647012461568 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚371698944752128 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚48218495 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 $ _timestamp‚1.784463638984882e+09 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚371698944752128 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚36299865 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚278343463855616 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4064830657024 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚367321096939984 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚10838786 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 3947979049472 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚86191611 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 trainer/global_step‚4799 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚371698927712768 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚37962971 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚48228638 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 3948067709440 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚25460906ˆuáÝ 2’- _wandbruntime‚3313  _runtime‚3313dRÒ‡ì&“æM G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚86209074 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3937912463333 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚371774077379072 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚25465958 _step‚393 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚48238472 % _timestamp‚1.7844636393910758e+09 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚86209074 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚36307045 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚37970602 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 3948853432320 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 150006272 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚36306872 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚371774077379072 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚169 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 127339520 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚371333282889349 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚37970600 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚10841083 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 _runtime‚3313.779698691 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚48238472 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚371774094418432 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚25465962 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚367395387465376 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3937999249871 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚37970602 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚282465025867264 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚375722842144256 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚371774094418432 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3948764765184 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚375722947850752 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3948764765184 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚86198915 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚37970600 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚173 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚282465247316480 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚48228315 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚375722947850752 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚86198915 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚10840914 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚371333386715247 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚36306872 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚371774077379072 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚169 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 127339520 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚371333282889349 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚37970600 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚10841083 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 _runtime‚3313.779698691 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚48238472 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚371774094418432 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚25465962 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚367395387465376 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3937999249871 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚37970602 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚282465025867264 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚375722842144256 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚371774094418432 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 3948764765184 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚375722947850752 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 3948764765184 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚86198915 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚37970600 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚173 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚282465247316480 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚48228315 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚375722947850752 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚86198915 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚10840914 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚371333386715247 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚371774077379072 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚25465958 _step‚393 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚48238472 % _timestamp‚1.7844636393910758e+09 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚86209074 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200‰Ž ·º2•- _wandbruntime‚3313  _runtime‚3313ÙýDÿ–ÿ 2loss/SRqqt1/assignment_loss‚0.7347019776633136 1loss/FBt2/assignment_loss‚0.16070736051486087 epoch‚11 _step‚394 2loss/SRqqt2/assignment_loss‚0.5914620922232826 1loss/FBt1/assignment_loss‚0.05743459232598039 /loss/FRt1/detection_loss‚0.5862526156016763 trainer/global_step‚4799 _runtime‚3313.780750493 /loss/FRt1/assignment_loss‚2.288942379765531 /loss/FRt2/detection_loss‚1.4264712682204708 1loss/SRqqt1/detection_loss‚1.0817401659971273 1loss/SRqqt2/detection_loss‚0.7700911606155686 /loss/FBt1/detection_loss‚0.5617747411253292 % _timestamp‚1.7844636393926358e+09 .loss/FBt2/detection_loss‚1.102129920552836 &loss/total_loss‚10.491778273581048 .loss/FRt2/assignment_loss‚1.13006999897507Ý9º– —„ _runtime‚3313.780750493 /loss/FRt1/assignment_loss‚2.288942379765531 /loss/FRt2/detection_loss‚1.4264712682204708 1loss/SRqqt1/detection_loss‚1.0817401659971273 1loss/SRqqt2/detection_loss‚0.7700911606155686 /loss/FBt1/detection_loss‚0.5617747411253292 % _timestamp‚1.7844636393926358e+09 .loss/FBt2/detection_loss‚1.102129920552836 &loss/total_loss‚10.491778273581048 .loss/FRt2/assignment_loss‚1.13006999897507 2loss/SRqqt1/assignment_loss‚0.7347019776633136 1loss/FBt2/assignment_loss‚0.16070736051486087 epoch‚11 _step‚394 2loss/SRqqt2/assignment_loss‚0.5914620922232826 1loss/FBt1/assignment_loss‚0.05743459232598039 /loss/FRt1/detection_loss‚0.5862526156016763 trainer/global_step‚4799Š:ˆíÿk˜‚(:à ˜‚óÒµ¤“¥& disk.nvme0c0n1.out‚34170.404296875( disk.nvme1c1n1.out‚15955.16455078125 disk.md0.in‚0 disk.md0.out‚81.5625# disk.dm-0.out‚48084.07275390625 disk.nvme1c1n1.in‚ 129.765625& disk.nvme1n1.out‚15955.16455078125# memory_percent‚99.3628978729248' proc.memory.availableMB‚ 104.3828125 proc.cpu.threads‚17 disk.nvme0n1.in‚ 1852.4453125) disk./.usagePercent‚76.05161780236385$ disk./.usageGB‚1167.582462310791 disk.rbd0.out‚0" disk.nvme0c0n1.in‚ 1852.4453125# proc.memory.rssMB‚ 2928.51953125 cpu‚25.89767845717172 disk.dm-1.in‚0! disk.dm-1.out‚1865.8173828125 disk.rbd0.in‚0$ disk.nvme0n1.out‚34170.404296875 disk.nvme1n1.in‚ 129.765625) proc.memory.percent‚17.87426471710205 network.sent‚ 7.0687e+06 network.recv‚879293 disk.dm-0.in‚ 1982.2109375$ê’ž™‚(:“ ˜‚óÒœ¶ø¥ gpu.0.gpu‚95.0 gpu.0.memory‚15, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚50.0 gpu.0.powerWatts‚192.818' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚64.27266666666667 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0 r‰2š- _wandbruntime‚3327  _runtime‚3327©Ú­šÓ ›Ê{ 9"Purity/2FRt2SRqqt0FBt/SRqqt_purity‚0.7990058660507202 *jet/accuracy_1_of_3‚0.9199216961860657 trainer/global_step‚4799 9"Purity/*FRt2SRqqt1FBt/SRqqt_purity‚0.3680555820465088 @&Purity/0FRt1SRqqt2FBt/event_proportion‚0.0004417782765813172 =&Purity/1FRt1SRqqt0FBt/event_proportion‚0.1467633992433548 8"Purity/2FRt*SRqqt*FBt/SRqqt_purity‚0.775068461894989 8 Purity/*FRt2SRqqt0FBt/FRt_purity‚0.06241176277399063 8 Purity/1FRt*SRqqt0FBt/FRt_purity‚0.12799304723739624 9"Purity/2FRt1SRqqt1FBt/SRqqt_purity‚0.7777777910232544 ,&Purity/2FRt2SRqqt1FBt/event_proportion‚0 /particle/accuracy_4_of_4‚0.9544184803962708 7 Purity/0FRt1SRqqt2FBt/FBt_purity‚0.9230769276618958 :"Purity/*FRt*SRqqt0FBt/event_purity‚0.23786631226539612 =&Purity/0FRt0SRqqt0FBt/event_proportion‚0.0462472103536129 ,&Purity/0FRt2SRqqt2FBt/event_proportion‚0 @&Purity/1FRt1SRqqt1FBt/event_proportion‚0.0064639137126505375 jet/accuracy_1_of_5‚1 jet/accuracy_5_of_5‚0 7 Purity/*FRt0SRqqt*FBt/FBt_purity‚0.9947766065597534 =&Purity/*FRt0SRqqt0FBt/event_proportion‚0.4607747495174408 7 Purity/*FRt0SRqqt1FBt/FRt_purity‚0.5807517170906067 :"Purity/*FRt2SRqqt0FBt/event_purity‚0.17023073136806488 =&Purity/1FRt*SRqqt0FBt/event_proportion‚0.6105840802192688 6validation_average_jet_accuracy‚0.3443704843521118 ) Purity/2FRt*SRqqt2FBt/FRt_purity‚0.75 8"Purity/0FRt1SRqqt1FBt/SRqqt_purity‚0.560309886932373 9"Purity/0FRt2SRqqt*FBt/SRqqt_purity‚0.6018019318580627 particle/accuracy_1_of_4‚1 =&Purity/*FRt0SRqqt2FBt/event_proportion‚0.0103004090487957 7 Purity/*FRt2SRqqt1FBt/FBt_purity‚0.9814814329147339 & Purity/1FRt*SRqqt1FBt/FBt_purity‚1 *jet/accuracy_1_of_2‚0.7790555357933044 ("Purity/*FRt1SRqqt2FBt/event_purity‚0 7 Purity/0FRt*SRqqt*FBt/FBt_purity‚0.9916927814483643 ("Purity/0FRt2SRqqt1FBt/event_purity‚0 :"Purity/1FRt0SRqqt0FBt/event_purity‚0.14319683611392975 =&Purity/2FRt*SRqqt0FBt/event_proportion‚0.0382486991584301 8 Purity/2FRt2SRqqt0FBt/FRt_purity‚0.06751606613397598 7 Purity/2FRt0SRqqt2FBt/FRt_purity‚0.7857142686843872 >&Purity/1FRt2SRqqt*FBt/event_proportion‚0.02322823740541935 6 Purity/*FRt0SRqqt2FBt/FRt_purity‚0.889266848564148 =&Purity/0FRt*SRqqt0FBt/event_proportion‚0.2322126179933548 9"Purity/0FRt1SRqqt0FBt/event_purity‚0.6397580504417419 8 Purity/1FRt0SRqqt*FBt/FRt_purity‚0.15689416229724884 @&Purity/1FRt0SRqqt2FBt/event_proportion‚0.0024879092816263437 & Purity/2FRt*SRqqt*FBt/FBt_purity‚1 7 Purity/2FRt*SRqqt1FBt/FRt_purity‚0.5041667222976685 7 Purity/*FRt*SRqqt1FBt/FBt_purity‚0.9969365000724792 :"Purity/*FRt2SRqqt*FBt/event_purity‚0.16710692644119263 9"Purity/0FRt1SRqqt1FBt/event_purity‚0.4790034890174866 8&Purity/1FRt2SRqqt0FBt/event_proportion‚ 0.02294921875 9"Purity/1FRt2SRqqt0FBt/SRqqt_purity‚0.7154293656349182 @&Purity/*FRt1SRqqt2FBt/event_proportion‚0.0006975446594879031 9"Purity/1FRt1SRqqt*FBt/SRqqt_purity‚0.7510365843772888 8 Purity/*FRt1SRqqt*FBt/FRt_purity‚0.12039189040660858 ?&Purity/0FRt*SRqqt2FBt/event_proportion‚0.008068266324698925 9"Purity/0FRt2SRqqt0FBt/SRqqt_purity‚0.6079427599906921 ."Purity/0FRt2SRqqt1FBt/SRqqt_purity‚0.40625 9"Purity/1FRt*SRqqt*FBt/SRqqt_purity‚0.7414253354072571 9"Purity/1FRt*SRqqt0FBt/SRqqt_purity‚0.7456483840942383 ("Purity/2FRt1SRqqt*FBt/event_purity‚0 'particle/f_score‚0.6353824138641357 :"Purity/0FRt2SRqqt0FBt/event_purity‚0.38488689064979553 :"Purity/1FRt*SRqqt0FBt/event_purÎi¬¸ý/ity‚0.10946587473154068 7 Purity/1FRt*SRqqt2FBt/FRt_purity‚0.8883219957351685 ("Purity/1FRt1SRqqt2FBt/event_purity‚0 =&Purity/2FRt0SRqqt*FBt/event_proportion‚0.0208565853536129 ?&Purity/*FRt*SRqqt2FBt/event_proportion‚0.010997953824698925 *jet/accuracy_1_of_4‚0.9683623313903809 particle/accuracy_1_of_3‚1 =&Purity/*FRt1SRqqt*FBt/event_proportion‚0.3866024911403656 ,&Purity/*FRt2SRqqt2FBt/event_proportion‚0 ("Purity/0FRt0SRqqt0FBt/event_purity‚1 ("Purity/1FRt2SRqqt0FBt/event_purity‚0 8 Purity/2FRt*SRqqt0FBt/FRt_purity‚0.13423307240009308 9"Purity/*FRt*SRqqt2FBt/event_purity‚0.8589910864830017 9"Purity/0FRt1SRqqt0FBt/SRqqt_purity‚0.6397580504417419 @&Purity/1FRt*SRqqt2FBt/event_proportion‚0.0027204242069274187 ("Purity/1FRt2SRqqt1FBt/event_purity‚0 :"Purity/2FRt*SRqqt1FBt/event_purity‚0.12999999523162842 9"Purity/0FRt*SRqqt2FBt/SRqqt_purity‚0.1538461595773697 *"Purity/2FRt*SRqqt2FBt/event_purity‚0.5 & Purity/2FRt0SRqqt1FBt/FBt_purity‚1 =&Purity/*FRt*SRqqt1FBt/event_proportion‚0.0617094486951828 7 Purity/*FRt*SRqqt2FBt/FRt_purity‚0.8820106983184814 7 Purity/1FRt0SRqqt2FBt/FBt_purity‚0.9970238208770752 epoch‚11 7 Purity/*FRt0SRqqt1FBt/FBt_purity‚0.9976010322570801 9"Purity/0FRt0SRqqt1FBt/event_purity‚0.9966605305671692 +jet/accuracy_2_of_2‚0.12912386655807495 ?&Purity/0FRt1SRqqt1FBt/event_proportion‚0.011555989272892475 :"Purity/1FRt2SRqqt1FBt/SRqqt_purity‚0.32499998807907104 9"Purity/2FRt*SRqqt1FBt/SRqqt_purity‚0.7777777910232544 particle/accuracy_1_of_1‚1 jet/accuracy_2_of_5‚1 particle/accuracy_1_of_5‚1 =&Purity/*FRt1SRqqt0FBt/event_proportion‚0.3670479953289032 7 Purity/*FRt1SRqqt2FBt/FBt_purity‚0.8379629254341125 ("Purity/0FRt1SRqqt2FBt/event_purity‚0 9"Purity/*FRt1SRqqt1FBt/SRqqt_purity‚0.6041334867477417 9"Purity/0FRt*SRqqt*FBt/event_purity‚0.6568453311920166 7 Purity/0FRt*SRqqt2FBt/FBt_purity‚0.9816230535507202 8 Purity/1FRt0SRqqt0FBt/FRt_purity‚0.14319683611392975 & Purity/1FRt1SRqqt1FBt/FBt_purity‚1 =&Purity/2FRt1SRqqt*FBt/event_proportion‚0.0127883180975914 *particle/specificity‚0.905982494354248 particle/accuracy_5_of_5‚0 :"Purity/*FRt*SRqqt*FBt/event_purity‚0.27515310049057007 +particle/sensitivity‚0.6257104277610779 ("Purity/2FRt1SRqqt2FBt/event_purity‚0 ("Purity/1FRt2SRqqt*FBt/event_purity‚0 & Purity/2FRt1SRqqt1FBt/FBt_purity‚1 particle/accuracy_2_of_5‚1 8 Purity/*FRt1SRqqt0FBt/FRt_purity‚0.10343560576438904 9"Purity/0FRt1SRqqt*FBt/event_purity‚0.6296303868293762 8"Purity/1FRt1SRqqt0FBt/SRqqt_purity‚0.755377471446991 =&Purity/*FRt*SRqqt*FBt/event_proportion‚0.9537528157234192 9"Purity/*FRt1SRqqt0FBt/SRqqt_purity‚0.6897456645965576 9"Purity/*FRt1SRqqt1FBt/event_purity‚0.3732053339481354 9"Purity/0FRt0SRqqt2FBt/event_purity‚0.9721544981002808 9"Purity/2FRt1SRqqt0FBt/SRqqt_purity‚0.7579745650291443 ,&Purity/2FRt2SRqqt2FBt/event_proportion‚0 particle/accuracy_2_of_3‚1 9"Purity/2FRt0SRqqt1FBt/event_purity‚0.2864583134651184 =&Purity/0FRt0SRqqt1FBt/event_proportion‚0.0313895083963871 7 Purity/0FRt0SRqqt1FBt/FBt_purity‚0.9966605305671692 , Purity/0FRt2SRqqt1FBt/FBt_purity‚0.96875 =&Purity/1FRt*SRqqt*FBt/event_proportion‚0.6297432780265808 9"Purity/1FRt0SRqqt1FBt/event_purity‚0.5015042424201965 >&Purity/1FRt0SRqqt1FBt/event_proportion‚0.00969587080180645 8 Purity/2FRt2SRqqt*FBt/FRt_purity‚0.06751606613397598 & Purity/2FRt*SRqqt2FBt/FBt_purity‚1 8"Purity/*FRt1SRqqt*FBt/event_purity‚0.372711718082428 >&Purity/1FRt*SRqqt1FBt/event_proportion‚0.01643880270421505 9"Purity/2FRt2SRqqt*FBt/SRqqt_purity‚0.7990058660507202 ,jet/accuracy_4_of_4‚0.013899499550461769 @&Purity/0FRt2SRqqt1FBt/event_proportion‚0.0007207961170934141 @&Purity/2FRt1SRqqt1FBt/event_proportion‚0.0008370535797439516 =&Purity/*FRt1SRqqt1FBt/event_proportion‚0.0188569575548172 ?&Purity/0FRt0SRqqt2FBt/event_proportion‚0.007626487873494625 8"Purity/1FRt*SRqqt1FBt/SRqqt_purity‚0.647337794303894 7 Purity/1FRt1SRqqt2FBt/FRt_purity‚0.8888888955116272 ;"Purity/2FRt*SRqqt*FBt/event_purity‚0.017962288111448288 particle/accuracy_1_of_2‚1 8"Purity/*FRt1SRqqt0FBt/event_purity‚0.373514324426651 <&Purity/*FRt2SRqqt*FBt/event_proportion‚0.054222472012043 8&Purity/*FRt2SRqqt0FBt/event_proportion‚ 0.05322265625 ;"Purity/2FRt0SRqqt0FBt/event_purity‚0.021202975884079933 ("Purity/2FRt1SRqqt0FBt/event_purity‚0 /particle/accuracy_2_of_2‚0.9998962879180908 9"Purity/*FRt0SRqqt1FBt/event_purity‚0.8681972026824951 ;"Purity/1FRt1SRqqt0FBt/event_purity‚0.025369301438331604 @&Purity/1FRt1SRqqt2FBt/event_proportion‚0.0002325148816453293 ( Purity/2FRt1SRqqt2FBt/FRt_purity‚0.5 <&Purity/*FRt*SRqqt0FBt/event_proportion‚0.881045401096344 9"Purity/1FRt*SRqqt1FBt/event_purity‚0.3911501169204712 9"Purity/1FRt*SRqqt2FBt/SRqqt_purity‚0.4444444477558136 *loss/val_total_loss‚10.060780652360586 7 Purity/*FRt1SRqqt*FBt/FBt_purity‚0.9860761761665344 8 Purity/*FRt1SRqqt1FBt/FRt_purity‚0.44935157895088196 =&Purity/0FRt*SRqqt1FBt/event_proportion‚0.0436662957072258 7 Purity/0FRt1SRqqt1FBt/FBt_purity‚0.9939466118812561 particle/accuracy_3_of_5‚1 7 Purity/1FRt0SRqqt*FBt/FBt_purity‚0.9988385438919067 7 Purity/1FRt0SRqqt1FBt/FRt_purity‚0.5893529057502747 & Purity/1FRt2SRqqt1FBt/FBt_purity‚1 8 Purity/2FRt*SRqqt*FBt/FRt_purity‚0.15162992477416992 9"Purity/*FRt0SRqqt2FBt/event_purity‚0.9192333221435547 :"Purity/*FRt0SRqqt*FBt/event_purity‚0.21318131685256958 9"Purity/*FRt2SRqqt*FBt/SRqqt_purity‚0.6691281795501709 & Purity/2FRt1SRqqt*FBt/FBt_purity‚1 7 Purity/2FRt1SRqqt1FBt/FRt_purity‚0.4560185670852661 =&Purity/*FRt0SRqqt1FBt/event_proportion‚0.0418526791036129 =&Purity/0FRt*SRqqt*FBt/event_proportion‚0.2839471697807312 7 Purity/1FRt*SRqqt2FBt/FBt_purity‚0.9755668640136719 :"Purity/*FRt0SRqqt0FBt/event_purity‚0.13789184391498566 9"Purity/1FRt*SRqqt*FBt/event_purity‚0.1194491982460022 7 Purity/1FRt1SRqqt*FBt/FBt_purity‚0.9838202595710754 5 Purity/1FRt1SRqqt1FBt/FRt_purity‚0.44171342253685 9 Purity/1FRt2SRqqt0FBt/FRt_purity‚0.057522643357515335 9"Purity/2FRt1SRqqt*FBt/SRqqt_purity‚0.7587220072746277 % _timestamp‚1.7844636531089656e+09 ;"Purity/1FRt1SRqqt*FBt/event_purity‚0.034355659037828445 8 Purity/2FRt0SRqqt*FBt/FRt_purity‚0.15790915489196777 particle/accuracy_4_of_5‚1 9"Purity/0FRt*SRqqt0FBt/SRqqt_purity‚0.6334764361381531 7 Purity/0FRt0SRqqt*FBt/FBt_purity‚0.9933624267578125 , Purity/0FRt2SRqqt*FBt/FBt_purity‚0.96875 9"Purity/1FRt*SRqqt2FBt/event_purity‚0.7274944186210632 7 Purity/*FRt*SRqqt2FBt/FBt_purity‚0.9803312420845032 8 Purity/*FRt0SRqqt*FBt/FRt_purity‚0.15692931413650513 :"Purity/0FRt2SRqqt*FBt/event_purity‚0.37392428517341614 7 Purity/1FRt0SRqqt2FBt/FRt_purity‚0.8896258473396301 @&Purity/2FRt*SRqqt1FBt/event_proportion‚0.0016043527284637094 jet/accuracy_3_of_5‚1 7 Purity/*FRt1SRqqt2FBt/FRt_purity‚0.8500000238418579 ?&Purity/2FRt2SRqqt*FBt/event_proportion‚0.006417410913854837 _runtime‚3327.497809992 particle/accuracy_2_of_4‚1 8 Purity/*FRt2SRqqt*FBt/FRt_purity‚0.06193951889872551 7 Purity/1FRt*SRqqt*FBt/FRt_purity‚0.1417386680841446 9"Purity/1FRt0SRqqt2FBt/event_purity‚0.8022675514221191 9"Purity/1FRt1SRqqt1FBt/SRqqt_purity‚0.6686124801635742 A&Purity/1FRt2SRqqt1FBt/event_proportion‚0.00027901786961592734 @&Purity/2FRt*SRqqt2FBt/event_proportion‚0.0002092633949359879 *jet/accuracy_2_of_3‚0.5107420086860657 ("Purity/2FRt1SRqqt2FBt/SRqqt_purity‚0 >&Purity/2FRt0SRqqt0FBt/event_proportion‚0.01990327425301075 A&Purity/2FRt0SRqqt2FBt/event_proportion‚0.00018601190822664648 9"Purity/0FRt*SRqqt0FBt/event_purity‚0.6130200624465942 9"Purity/0FRt0SRqqt*FBt/event_purity‚0.9919893145561218 =&Purity/0FRt1SRqqt0FBt/event_proportion‚0.2083565890789032 @&Purity/2FRt1SRqqt2FBt/event_proportion‚2.325148852833081e-05 & Purity/2FRt0SRqqt2FBt/FBt_purity‚1 & Purity/1FRt0SRqqt1FBt/FBt_purity‚1 8 Purity/1FRt1SRqqt0FBt/FRt_purity‚0.09294021874666214 :"Purity/1FRt1SRqqt1FBt/event_purity‚0.23741447925567627 & Purity/1FRt2SRqqt*FBt/FBt_purity‚1 & Purity/1FRt2SRqqt1FBt/FRt_purity‚0 ?&Purity/2FRt2SRqqt0FBt/event_proportion‚0.006417410913854837 7 Purity/*FRt*SRqqt*FBt/FRt_purity‚0.1428685486316681 7 Purity/*FRt0SRqqt2FBt/FBt_purity‚0.9887163043022156 =&Purity/0FRt0SRqqt*FBt/event_proportion‚0.0390159972012043 9"Purity/1FRt0SRqqt*FBt/event_purity‚0.1544291228055954 +jet/accuracy_3_of_4‚0.08315278589725494 (particle/accuracy‚0.8457962274551392 _step‚395 =&Purity/1FRt0SRqqt*FBt/event_proportion‚0.4530552327632904 6 Purity/2FRt1SRqqt0FBt/FRt_purity‚0.165994793176651 7 Purity/0FRt1SRqqt*FBt/FBt_purity‚0.9877076148986816 7 Purity/*FRt1SRqqt1FBt/FBt_purity‚0.9962709546089172 =&Purity/1FRt0SRqqt0FBt/event_proportion‚0.4408714771270752 9"Purity/*FRt2SRqqt0FBt/SRqqt_purity‚0.6746821999549866 7 Purity/0FRt*SRqqt1FBt/FBt_purity‚0.9955921173095703 9"Purity/1FRt1SRqqt2FBt/SRqqt_purity‚0.4444444477558136 7 Purity/1FRt1SRqqt2FBt/FBt_purity‚0.7222222089767456 +jet/accuracy_3_of_3‚0.06384018808603287 9"Purity/*FRt*SRqqt0FBt/SRqqt_purity‚0.6862538456916809 8 Purity/*FRt0SRqqt0FBt/FRt_purity‚0.14265155792236328 9"Purity/*FRt*SRqqt1FBt/SRqqt_purity‚0.5818271636962891 9"Purity/*FRt1SRqqt*FBt/SRqqt_purity‚0.6846860647201538 9"Purity/0FRt*SRqqt1FBt/SRqqt_purity‚0.5418774485588074 =&Purity/0FRt1SRqqt*FBt/event_proportion‚0.2203543484210968 ;"Purity/2FRt0SRqqt*FBt/event_purity‚0.034956831485033035 ("Purity/2FRt2SRqqt*FBt/event_purity‚0 *jet/accuracy_2_of_4‚0.7551811933517456 & Purity/2FRt1SRqqt2FBt/FBt_purity‚1 :"Purity/*FRt*SRqqt2FBt/SRqqt_purity‚0.26851850748062134 7 Purity/*FRt2SRqqt*FBt/FBt_purity‚0.9814814329147339 @&Purity/*FRt2SRqqt1FBt/event_proportion‚0.0009998140158131719 & Purity/*FRt2SRqqt1FBt/FRt_purity‚0 9"Purity/0FRt*SRqqt2FBt/event_purity‚0.9152169227600098 7 Purity/1FRt*SRqqt*FBt/FBt_purity‚0.9932025074958801 7 Purity/1FRt*SRqqt1FBt/FRt_purity‚0.5264426469802856 ,&Purity/1FRt2SRqqt2FBt/event_proportion‚0 9"Purity/*FRt*SRqqt*FBt/SRqqt_purity‚0.6811689734458923 =&Purity/1FRt1SRqqt*FBt/event_proportion‚0.1534598171710968 <&Purity/2FRt*SRqqt*FBt/event_proportion‚0.040062315762043 ("Purity/2FRt2SRqqt0FBt/event_purity‚0 particle/accuracy_3_of_3‚1 /particle/accuracy_3_of_4‚0.9912415742874146 ("Purity/2FRt*SRqqt2FBt/SRqqt_purity‚0 9"Purity/0FRt*SRqqt*FBt/SRqqt_purity‚0.6280620098114014 @&Purity/2FRt0SRqqt1FBt/event_proportion‚0.0007672990905120969 7 Purity/0FRt0SRqqt2FBt/FBt_purity‚0.9860772490501404 9"Purity/0FRt1SRqqt*FBt/SRqqt_purity‚0.6343977451324463 >&Purity/0FRt2SRqqt0FBt/event_proportion‚0.02385602705180645 8 Purity/1FRt2SRqqt*FBt/FRt_purity‚0.05684543773531914 :"Purity/2FRt*SRqqt0FBt/event_purity‚0.01087874360382557 & Purity/2FRt*SRqqt1FBt/FBt_purity‚1 jet/accuracy_4_of_5‚0 7 Purity/*FRt*SRqqt1FBt/FRt_purity‚0.5196805000305176 7 Purity/*FRt*SRqqt*FBt/FBt_purity‚0.9925275444984436 9"Purity/*FRt*SRqqt1FBt/event_purity‚0.7023404240608215 9"Purity/0FRt*SRqqt1FBt/event_purity‚0.8406592607498169 9"Purity/1FRt2SRqqt*FBt/SRqqt_purity‚0.7102972269058228 7 Purity/2FRt0SRqqt0FBt/FRt_purity‚0.1369171440601349 9"Purity/0FRt1SRqqt2FBt/SRqqt_purity‚0.1538461595773697 =&Purity/*FRt0SRqqt*FBt/event_proportion‚0.5129278302192688 >&Purity/0FRt2SRqqt*FBt/event_proportion‚0.02457682229578495 8 Purity/1FRt1SRqqt*FBt/FRt_purity‚0.10949503630399704 ("Purity/2FRt1SRqqt1FBt/event_purity‚0 *jet/accuracy_1_of_1‚0.2975258231163025 9"Purity/2FRt0SRqqt2FBt/event_purity‚0.5714285969734192 8 Purity/*FRt*SRqqt0FBt/FRt_purity‚0.12872615456581116 ("Purity/*FRt2SRqqt1FBt/event_purity‚0 9"Purity/2FRt*SRqqt0FBt/SRqqt_purity‚0.7756356596946716 & Purity/2FRt0SRqqt*FBt/FBt_purity‚1 + Purity/2FRt0SRqqt1FBt/FRt_purity‚0.5625 :"Purity/*FRt1SRqqt2FBt/SRqqt_purity‚0.26851850748062134 7 Purity/2FRt1SRqqt*FBt/FRt_purity‚0.1841183453798294 ?&Purity/2FRt1SRqqt0FBt/event_proportion‚0.011928013525903225d&[¸Õ=œÏ{ & Purity/1FRt1SRqqt1FBt/FBt_purity‚1 =&Purity/2FRt1SRqqt*FBt/event_proportion‚0.0127883180975914 *particle/specificity‚0.905982494354248 particle/accuracy_5_of_5‚0 :"Purity/*FRt*SRqqt*FBt/event_purity‚0.27515310049057007 +particle/sensitivity‚0.6257104277610779 ("Purity/2FRt1SRqqt2FBt/event_purity‚0 ("Purity/1FRt2SRqqt*FBt/event_purity‚0 & Purity/2FRt1SRqqt1FBt/FBt_purity‚1 particle/accuracy_2_of_5‚1 8 Purity/*FRt1SRqqt0FBt/FRt_purity‚0.10343560576438904 9"Purity/0FRt1SRqqt*FBt/event_purity‚0.6296303868293762 8"Purity/1FRt1SRqqt0FBt/SRqqt_purity‚0.755377471446991 =&Purity/*FRt*SRqqt*FBt/event_proportion‚0.9537528157234192 9"Purity/*FRt1SRqqt0FBt/SRqqt_purity‚0.6897456645965576 9"Purity/*FRt1SRqqt1FBt/event_purity‚0.3732053339481354 9"Purity/0FRt0SRqqt2FBt/event_purity‚0.9721544981002808 9"Purity/2FRt1SRqqt0FBt/SRqqt_purity‚0.7579745650291443 ,&Purity/2FRt2SRqqt2FBt/event_proportion‚0 particle/accuracy_2_of_3‚1 9"Purity/2FRt0SRqqt1FBt/event_purity‚0.2864583134651184 =&Purity/0FRt0SRqqt1FBt/event_proportion‚0.0313895083963871 7 Purity/0FRt0SRqqt1FBt/FBt_purity‚0.9966605305671692 , Purity/0FRt2SRqqt1FBt/FBt_purity‚0.96875 =&Purity/1FRt*SRqqt*FBt/event_proportion‚0.6297432780265808 9"Purity/1FRt0SRqqt1FBt/event_purity‚0.5015042424201965 >&Purity/1FRt0SRqqt1FBt/event_proportion‚0.00969587080180645 8 Purity/2FRt2SRqqt*FBt/FRt_purity‚0.06751606613397598 & Purity/2FRt*SRqqt2FBt/FBt_purity‚1 8"Purity/*FRt1SRqqt*FBt/event_purity‚0.372711718082428 >&Purity/1FRt*SRqqt1FBt/event_proportion‚0.01643880270421505 9"Purity/2FRt2SRqqt*FBt/SRqqt_purity‚0.7990058660507202 ,jet/accuracy_4_of_4‚0.013899499550461769 @&Purity/0FRt2SRqqt1FBt/event_proportion‚0.0007207961170934141 @&Purity/2FRt1SRqqt1FBt/event_proportion‚0.0008370535797439516 =&Purity/*FRt1SRqqt1FBt/event_proportion‚0.0188569575548172 ?&Purity/0FRt0SRqqt2FBt/event_proportion‚0.007626487873494625 8"Purity/1FRt*SRqqt1FBt/SRqqt_purity‚0.647337794303894 7 Purity/1FRt1SRqqt2FBt/FRt_purity‚0.8888888955116272 ;"Purity/2FRt*SRqqt*FBt/event_purity‚0.017962288111448288 particle/accuracy_1_of_2‚1 8"Purity/*FRt1SRqqt0FBt/event_purity‚0.373514324426651 <&Purity/*FRt2SRqqt*FBt/event_proportion‚0.054222472012043 8&Purity/*FRt2SRqqt0FBt/event_proportion‚ 0.05322265625 ;"Purity/2FRt0SRqqt0FBt/event_purity‚0.021202975884079933 ("Purity/2FRt1SRqqt0FBt/event_purity‚0 /particle/accuracy_2_of_2‚0.9998962879180908 9"Purity/*FRt0SRqqt1FBt/event_purity‚0.8681972026824951 ;"Purity/1FRt1SRqqt0FBt/event_purity‚0.025369301438331604 @&Purity/1FRt1SRqqt2FBt/event_proportion‚0.0002325148816453293 ( Purity/2FRt1SRqqt2FBt/FRt_purity‚0.5 <&Purity/*FRt*SRqqt0FBt/event_proportion‚0.881045401096344 9"Purity/1FRt*SRqqt1FBt/event_purity‚0.3911501169204712 9"Purity/1FRt*SRqqt2FBt/SRqqt_purity‚0.4444444477558136 *loss/val_total_loss‚10.060780652360586 7 Purity/*FRt1SRqqt*FBt/FBt_purity‚0.9860761761665344 8 Purity/*FRt1SRqqt1FBt/FRt_purity‚0.44935157895088196 =&Purity/0FRt*SRqqt1FBt/event_proportion‚0.0436662957072258 7 Purity/0FRt1SRqqt1FBt/FBt_purity‚0.9939466118812561 particle/accuracy_3_of_5‚1 7 Purity/1FRt0SRqqt*FBt/FBt_purity‚0.9988385438919067 7 Purity/1FRt0SRqqt1FBt/FRt_purity‚0.5893529057502747 & Purity/1FRt2SRqqt1FBt/FBt_purity‚1 8 Purity/2FRt*SRqqt*FBt/FRt_purity‚0.15162992477416992 9"Purity/*FRt0SRqqt2FBt/event_purity‚0.9192333221435547 :"Purity/*FRt0SRqqt*FBt/event_purity‚0.21318131685256958 9"Purity/*FRt2SRqqt*FBt/SRqqt_purity‚0.6691281795501709 & Purity/2FRt1SRqqt*FBt/FBt_purity‚1 7 Purity/2FRt1SRqqt1FBt/FRt_purity‚0.4560185670852661 =&Purity/*FRt0SRqqt1FBt/event_proportion‚0.0418526791036129 =&Purity/0FRt*SRqqt*FBt/event_proportion‚0.2839471697807312 7 Purity/1FRt*SRqqt2FBt/FBt_purity‚0.9755668640136719 :"Purity/*FRt0SRqqt0FBt/event_purity‚0.13789184391498566 9"Purity/1FRt*SRqqt*FBt/event_purity‚0.1194491982460022 7 Purity/1FRt1SRqqt*FBt/FBt_purity‚0.9838202595710754 5 Purity/1FRt1SRqqt1FBt/FRt_purity‚0.44171342253685 9 Purity/1FRt2SRqqt0FBt/FRt_purity‚0.057522643357515335 9"Purity/2FRt1SRqqt*FBt/SRqqt_purity‚0.7587220072746277 % _timestamp‚1.7844636531089656e+09 ;"Purity/1FRt1SRqqt*FBt/event_purity‚0.034355659037828445 8 Purity/2FRt0SRqqt*FBt/FRt_purity‚0.15790915489196777 particle/accuracy_4_of_5‚1 9"Purity/0FRt*SRqqt0FBt/SRqqt_purity‚0.6334764361381531 7 Purity/0FRt0SRqqt*FBt/FBt_purity‚0.9933624267578125 , Purity/0FRt2SRqqt*FBt/FBt_purity‚0.96875 9"Purity/1FRt*SRqqt2FBt/event_purity‚0.7274944186210632 7 Purity/*FRt*SRqqt2FBt/FBt_purity‚0.9803312420845032 8 Purity/*FRt0SRqqt*FBt/FRt_purity‚0.15692931413650513 :"Purity/0FRt2SRqqt*FBt/event_purity‚0.37392428517341614 7 Purity/1FRt0SRqqt2FBt/FRt_purity‚0.8896258473396301 @&Purity/2FRt*SRqqt1FBt/event_proportion‚0.0016043527284637094 jet/accuracy_3_of_5‚1 7 Purity/*FRt1SRqqt2FBt/FRt_purity‚0.8500000238418579 ?&Purity/2FRt2SRqqt*FBt/event_proportion‚0.006417410913854837 _runtime‚3327.497809992 particle/accuracy_2_of_4‚1 8 Purity/*FRt2SRqqt*FBt/FRt_purity‚0.06193951889872551 7 Purity/1FRt*SRqqt*FBt/FRt_purity‚0.1417386680841446 9"Purity/1FRt0SRqqt2FBt/event_purity‚0.8022675514221191 9"Purity/1FRt1SRqqt1FBt/SRqqt_purity‚0.6686124801635742 A&Purity/1FRt2SRqqt1FBt/event_proportion‚0.00027901786961592734 @&Purity/2FRt*SRqqt2FBt/event_proportion‚0.0002092633949359879 *jet/accuracy_2_of_3‚0.5107420086860657 ("Purity/2FRt1SRqqt2FBt/SRqqt_purity‚0 >&Purity/2FRt0SRqqt0FBt/event_proportion‚0.01990327425301075 A&Purity/2FRt0SRqqt2FBt/event_proportion‚0.00018601190822664648 9"Purity/0FRt*SRqqt0FBt/event_purity‚0.6130200624465942 9"Purity/0FRt0SRqqt*FBt/event_purity‚0.9919893145561218 =&Purity/0FRt1SRqqt0FBt/event_proportion‚0.2083565890789032 @&Purity/2FRt1SRqqt2FBt/event_proportion‚2.325148852833081e-05 & Purity/2FRt0SRqqt2FBt/FBt_purity‚1 & Purity/1FRt0SRqqt1FBt/FBt_purity‚1 8 Purity/1FRt1SRqqt0FBt/FRt_purity‚0.09294021874666214 :"Purity/1FRt1SRqqt1FBt/event_purity‚0.23741447925567627 & Purity/1FRt2SRqqt*FBt/FBt_purity‚1 & Purity/1FRt2SRqqt1FBt/FRt_purity‚0 ?&Purity/2FRt2SRqqt0FBt/event_proportion‚0.006417410913854837 7 Purity/*FRt*SRqqt*FBt/FRt_purity‚0.1428685486316681 7 Purity/*FRt0SRqqt2FBt/FBt_purity‚0.9887163043022156 =&Purity/0FRt0SRqqt*FBt/event_proportion‚0.0390159972012043 9"Purity/1FRt0SRqqt*FBt/event_purity‚0.1544291228055954 +jet/accuracy_3_of_4‚0.08315278589725494 (particle/accuracy‚0.8457962274551392 _step‚395 =&Purity/1FRt0SRqqt*FBt/event_proportion‚0.4530552327632904 6 Purity/2FRt1SRqqt0FBt/FRt_purity‚0.165994793176651 7 Purity/0FRt1SRqqt*FBt/FBt_purity‚0.9877076148986816 7 Purity/*FRt1SRqqt1FBt/FBt_purity‚0.9962709546089172 =&Purity/1FRt0SRqqt0FBt/event_proportion‚0.4408714771270752 9"Purity/*FRt2SRqqt0FBt/SRqqt_purity‚0.6746821999549866 7 Purity/0FRt*SRqqt1FBt/FBt_purity‚0.9955921173095703 9"Purity/1FRt1SRqqt2FBt/SRqqt_purity‚0.4444444477558136 7 Purity/1FRt1SRqqt2FBt/FBt_purity‚0.7222222089767456 +jet/accuracy_3_of_3‚0.06384018808603287 9"Purity/*FRt*SRqqt0FBt/SRqqt_purity‚0.6862538456916809 8 Purity/*FRt0SRqqt0FBt/FRt_purity‚0.14265155792236328 9"Purity/*FRt*SRqqt1FBt/SRqqt_purity‚0.5818271636962891 9"Purity/*FRt1SRqqt*FBt/SRqqt_purity‚0.6846860647201538 9"Purity/0FRt*SRqqt1FBt/SRqqt_purity‚0.5418774485588074 =&Purity/0FRt1SRqqt*FBt/event_proportion‚0.2203543484210968 ;"Purity/2FRt0SRqqt*FBt/event_purity‚0.034956831485033035 ("Purity/2FRt2SRqqt*FBt/event_purity‚0 *jet/accuracy_2_of_4‚0.7551811933517456 & Purity/2FRt1SRqqt2FBt/FBt_purity‚1 :"Purity/*FRt*SRqqt2FBt/SRqqt_purity‚0.26851850748062134 7 Purity/*FRt2SRqqt*FBt/FBt_purity‚0.9814814329147339 @&Purity/*FRt2SRqqt1FBt/event_proportion‚0.0009998140158131719 & Purity/*FRt2SRqqt1FBt/FRt_purity‚0 9"Purity/0FRt*SRqqt2FBt/event_purity‚0.9152169227600098 7 Purity/1FRt*SRqqt*FBt/FBt_purity‚0.9932025074958801 7 Purity/1FRt*SRqqt1FBt/FRt_purity‚0.5264426469802856 ,&Purity/1FRt2SRqqt2FBt/event_proportion‚0 9"Purity/*FRt*SRqqt*FBt/SRqqt_purity‚0.6811689734458923 =&Purity/1FRt1SRqqt*FBt/event_proportion‚0.1534598171710968 <&Purity/2FRt*SRqqt*FBt/event_proportion‚0.040062315762043 ("Purity/2FRt2SRqqt0FBt/event_purity‚0 particle/accuracy_3_of_3‚1 /particle/accuracy_3_of_4‚0.9912415742874146 ("Purity/2FRt*SRqqt2FBt/SRqqt_purity‚0 9"Purity/0FRt*SRqqt*FBt/SRqqt_purity‚0.6280620098114014 @&Purity/2FRt0SRqqt1FBt/event_proportion‚0.0007672990905120969 7 Purity/0FRt0SRqqt2FBt/FBt_purity‚0.9860772490501404 9"Purity/0FRt1SRqqt*FBt/SRqqt_purity‚0.6343977451324463 >&Purity/0FRt2SRqqt0FBt/event_proportion‚0.02385602705180645 8 Purity/1FRt2SRqqt*FBt/FRt_purity‚0.05684543773531914 :"Purity/2FRt*SRqqt0FBt/event_purity‚0.01087874360382557 & Purity/2FRt*SRqqt1FBt/FBt_purity‚1 jet/accuracy_4_of_5‚0 7 Purity/*FRt*SRqqt1FBt/FRt_purity‚0.5196805000305176 7 Purity/*FRt*SRqqt*FBt/FBt_purity‚0.9925275444984436 9"Purity/*FRt*SRqqt1FBt/event_purity‚0.7023404240608215 9"Purity/0FRt*SRqqt1FBt/event_purity‚0.8406592607498169 9"Purity/1FRt2SRqqt*FBt/SRqqt_purity‚0.7102972269058228 7 Purity/2FRt0SRqqt0FBt/FRt_purity‚0.1369171440601349 9"Purity/0FRt1SRqqt2FBt/SRqqt_purity‚0.1538461595773697 =&Purity/*FRt0SRqqt*FBt/event_proportion‚0.5129278302192688 >&Purity/0FRt2SRqqt*FBt/event_proportion‚0.02457682229578495 8 Purity/1FRt1SRqqt*FBt/FRt_purity‚0.10949503630399704 ("Purity/2FRt1SRqqt1FBt/event_purity‚0 *jet/accuracy_1_of_1‚0.2975258231163025 9"Purity/2FRt0SRqqt2FBt/event_purity‚0.5714285969734192 8 Purity/*FRt*SRqqt0FBt/FRt_purity‚0.12872615456581116 ("Purity/*FRt2SRqqt1FBt/event_purity‚0 9"Purity/2FRt*SRqqt0FBt/SRqqt_purity‚0.7756356596946716 & Purity/2FRt0SRqqt*FBt/FBt_purity‚1 + Purity/2FRt0SRqqt1FBt/FRt_purity‚0.5625 :"Purity/*FRt1SRqqt2FBt/SRqqt_purity‚0.26851850748062134 7 Purity/2FRt1SRqqt*FBt/FRt_purity‚0.1841183453798294 ?&Purity/2FRt1SRqqt0FBt/event_proportion‚0.011928013525903225 9"Purity/2FRt2SRqqt0FBt/SRqqt_purity‚0.7990058660507202 *jet/accuracy_1_of_3‚0.9199216961860657 trainer/global_step‚4799 9"Purity/*FRt2SRqqt1FBt/SRqqt_purity‚0.3680555820465088 @&Purity/0FRt1SRqqt2FBt/event_proportion‚0.0004417782765813172 =&Purity/1FRt1SRqqt0FBt/event_proportion‚0.1467633992433548 8"Purity/2FRt*SRqqt*FBt/SRqqt_purity‚0.775068461894989 8 Purity/*FRt2SRqqt0FBt/FRt_purity‚0.06241176277399063 8 Purity/1FRt*SRqqt0FBt/FRt_purity‚0.12799304723739624 9"Purity/2FRt1SRqqt1FBt/SRqqt_purity‚0.7777777910232544 ,&Purity/2FRt2SRqqt1FBt/event_proportion‚0 /particle/accuracy_4_of_4‚0.9544184803962708 7 Purity/0FRt1SRqqt2FBt/FBt_purity‚0.9230769276618958 :"Purity/*FRt*SRqqt0FBt/event_purity‚0.23786631226539612 =&Purity/0FRt0SRqqt0FBt/event_proportion‚0.0462472103536129 ,&Purity/0FRt2SRqqt2FBt/event_proportion‚0 @&Purity/1FRt1SRqqt1FBt/event_proportion‚0.0064639137126505375 jet/accuracy_1_of_5‚1 jet/accuracy_5_of_5‚0 7 Purity/*FRt0SRqqt*FBt/FBt_purity‚0.9947766065597534 =&Purity/*FRt0SRqqt0FBt/event_proportion‚0.4607747495174408 7 Purity/*FRt0SRqqt1FBt/FRt_purity‚0.5807517170906067 :"Purity/*FRt2SRqqt0FBt/event_purity‚0.17023073136806488 =&Purity/1FRt*SRqqt0FBt/event_proportion‚0.6105840802192688 6validation_average_jet_accuracy‚0.3443704843521118 ) Purity/2FRt*SRqqt2FBt/FRt_purity‚0.75 8"Purity/0FRt1SRqqt1FBt/SRqqt_purity‚0.560309886932373 9"Purity/0FRt2SRqqt*FBt/SRqqt_purity‚0.6018019318580627 particle/accuracy_1_of_4‚1 =&Purity/*FRt0SRqqt2FBt/event_proportion‚0.0103004090487957 7 Purity/*FRt2SRqqt1FBt/FBt_purity‚0.9814814329147339 & Purity/1FRt*SRqqt1FBt/FBt_purity‚1 *jet/accuracy_1_of_2‚0.7790555357933044 ("Purity/*FRt1SRqqt2FBt/event_purity‚0 7 Purity/0FRt*SRqqt*FBt/FBt_purity‚0.9916927814483643 ("Purity/0FRt2SRqqt1FBt/event_purity‚0 :"Purity/1FRt0SRqqt0FBt/event_purity‚0.14319683611392975 =&Purity/2FRt*SRqqt0FBt/event_proportion‚0.0382486991584301 8 Purity/2FRt2SRqqt0FBt/FRt_purity‚0.06751606613397598 7 Purity/2FRt0SRqqt2FBt/FRt_purity‚0.7857142686843872 >&Purity/1FRt2SRqqt*FBt/event_proportion‚0.02322823740541935 6 Purity/*FRt0SRqqt2FBt/FRt_purity‚0.889266848564148 =&Purity/0FRt*SRqqt0FBt/event_proportion‚0.2322126179933548 9"Purity/0FRt1SRqqt0FBt/event_purity‚0.6397580504417419 8 Purity/1FRt0SRqqt*FBt/FRt_purity‚0.15689416229724884 @&Purity/1FRt0SRqqt2FBt/event_proportion‚0.0024879092816263437 & Purity/2FRt*SRqqt*FBt/FBt_purity‚1 7 Purity/2FRt*SRqqt1FBt/FRt_purity‚0.5041667222976685 7 Purity/*FRt*SRqqt1FBt/FBt_purity‚0.9969365000724792 :"Purity/*FRt2SRqqt*FBt/event_purity‚0.16710692644119263 9"Purity/0FRt1SRqqt1FBt/event_purity‚0.4790034890174866 8&Purity/1FRt2SRqqt0FBt/event_proportion‚ 0.02294921875 9"Purity/1FRt2SRqqt0FBt/SRqqt_purity‚0.7154293656349182 @&Purity/*FRt1SRqqt2FBt/event_proportion‚0.0006975446594879031 9"Purity/1FRt1SRqqt*FBt/SRqqt_purity‚0.7510365843772888 8 Purity/*FRt1SRqqt*FBt/FRt_purity‚0.12039189040660858 ?&Purity/0FRt*SRqqt2FBt/event_proportion‚0.008068266324698925 9"Purity/0FRt2SRqqt0FBt/SRqqt_purity‚0.6079427599906921 ."Purity/0FRt2SRqqt1FBt/SRqqt_purity‚0.40625 9"Purity/1FRt*SRqqt*FBt/SRqqt_purity‚0.7414253354072571 9"Purity/1FRt*SRqqt0FBt/SRqqt_purity‚0.7456483840942383 ("Purity/2FRt1SRqqt*FBt/event_purity‚0 'particle/f_score‚0.6353824138641357 :"Purity/0FRt2SRqqt0FBt/event_purity‚0.38488689064979553 :"Purity/1FRt*SRqqt0FBt/event_purity‚0.10946587473154068 7 Purity/1FRt*SRqqt2FBt/FRt_purity‚0.8883219957351685 ("Purity/1FRt1SRqqt2FBt/event_purity‚0 =&Purity/2FRt0SRqqt*FBt/event_proportion‚0.0208565853536129 ?&Purity/*FRt*SRqqt2FBt/event_proportion‚0.010997953824698925 *jet/accuracy_1_of_4‚0.9683623313903809 particle/accuracy_1_of_3‚1 =&Purity/*FRt1SRqqt*FBt/event_proportion‚0.3866024911403656 ,&Purity/*FRt2SRqqt2FBt/event_proportion‚0 ("Purity/0FRt0SRqqt0FBt/event_purity‚1 ("Purity/1FRt2SRqqt0FBt/event_purity‚0 8 Purity/2FRt*SRqqt0FBt/FRt_purity‚0.13423307240009308 9"Purity/*FRt*SRqqt2FBt/event_purity‚0.8589910864830017 9"Purity/0FRt1SRqqt0FBt/SRqqt_purity‚0.6397580504417419 @&Purity/1FRt*SRqqt2FBt/event_proportion‚0.0027204242069274187 ("Purity/1FRt2SRqqt1FBt/event_purity‚0 :"Purity/2FRt*SRqqt1FBt/event_purity‚0.12999999523162842 9"Purity/0FRt*SRqqt2FBt/SRqqt_purity‚0.1538461595773697 *"Purity/2FRt*SRqqt2FBt/event_purity‚0.5 & Purity/2FRt0SRqqt1FBt/FBt_purity‚1 =&Purity/*FRt*SRqqt1FBt/event_proportion‚0.0617094486951828 7 Purity/*FRt*SRqqt2FBt/FRt_purity‚0.8820106983184814 7 Purity/1FRt0SRqqt2FBt/FBt_purity‚0.9970238208770752 epoch‚11 7 Purity/*FRt0SRqqt1FBt/FBt_purity‚0.9976010322570801 9"Purity/0FRt0SRqqt1FBt/event_purity‚0.9966605305671692 +jet/accuracy_2_of_2‚0.12912386655807495 ?&Purity/0FRt1SRqqt1FBt/event_proportion‚0.011555989272892475 :"Purity/1FRt2SRqqt1FBt/SRqqt_purity‚0.32499998807907104 9"Purity/2FRt*SRqqt1FBt/SRqqt_purity‚0.7777777910232544 particle/accuracy_1_of_1‚1 jet/accuracy_2_of_5‚1 particle/accuracy_1_of_5‚1 =&Purity/*FRt1SRqqt0FBt/event_proportion‚0.3670479953289032 7 Purity/*FRt1SRqqt2FBt/FBt_purity‚0.8379629254341125 ("Purity/0FRt1SRqqt2FBt/event_purity‚0 9"Purity/*FRt1SRqqt1FBt/SRqqt_purity‚0.6041334867477417 9"Purity/0FRt*SRqqt*FBt/event_purity‚0.6568453311920166 7 Purity/0FRt*SRqqt2FBt/FBt_purity‚0.9816230535507202 8 Purity/1FRt0SRqqt0FBt/FRt_purity‚0.14319683611392975‹±ÑøŸ“‚(:ˆ §‚óÒ½—¿¤ gpu.0.gpu‚98.0 gpu.0.memory‚23, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚49.0 gpu.0.powerWatts‚221.829' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚73.943 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0äoùqž‚(:æ §‚óÒ¨ñ‹¥ disk.md0.in‚0! disk.dm-1.out‚1929.8369140625 disk.rbd0.out‚0" disk.nvme0c0n1.in‚ 1852.4453125& disk.nvme0c0n1.out‚34362.486328125$ memory_percent‚99.20709133148193* proc.memory.percent‚17.773032188415527 network.recv‚883117 disk.rbd0.in‚0( proc.memory.availableMB‚ 129.91015625# proc.memory.rssMB‚ 2911.93359375% disk.nvme1n1.out‚16075.7666015625 disk.md0.out‚ 81.82421875 disk.dm-0.in‚ 1982.2109375! disk.dm-0.out‚48332.158203125 cpu‚25.907933472601055 proc.cpu.threads‚17 network.sent‚ 7.144924e+06( disk./.usagePercent‚76.0650677006207% disk./.usageGB‚1167.7889518737793 disk.nvme0n1.in‚ 1852.4453125$ disk.nvme0n1.out‚34362.486328125 disk.nvme1c1n1.in‚ 129.765625' disk.nvme1c1n1.out‚16075.7666015625 disk.dm-1.in‚0 disk.nvme1n1.in‚ 129.765625ˆmežŸ‚(:“ ¶‚óÒïÏð¥ gpu.0.gpu‚31.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚48.0 gpu.0.powerWatts‚113.904' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚37.967999999999996 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ËQïl ‚(:á ¶‚óÒ¦î¦ disk.rbd0.in‚0 disk.rbd0.out‚0& disk.nvme0n1.out‚34363.17724609375 disk.md0.in‚0! disk.dm-0.out‚48332.744140625# proc.memory.rssMB‚ 2911.93359375 proc.cpu.threads‚17 network.sent‚ 7.189464e+06 disk.nvme0n1.in‚ 1852.4453125 disk.nvme1c1n1.in‚ 129.765625 disk.nvme1n1.in‚ 129.765625 disk.dm-1.in‚0" disk.dm-1.out‚1930.01220703125* proc.memory.percent‚17.773032188415527( disk./.usagePercent‚76.0650763972111% disk./.usageGB‚1167.7890853881836" disk.nvme0c0n1.in‚ 1852.4453125( disk.nvme0c0n1.out‚34363.17724609375! disk.nvme1n1.out‚ 16076.359375# memory_percent‚99.2072343826294( proc.memory.availableMB‚ 129.88671875 cpu‚25.90914036177003 network.recv‚886282# disk.nvme1c1n1.out‚ 16076.359375 disk.md0.out‚ 82.06640625 disk.dm-0.in‚ 1982.2109375X| Y2¡- _wandbruntime‚3359  _runtime‚3359Kn{ɢà ' lr-AdamW/pg1‚0.00012227806371455693 ' lr-AdamW/pg2‚0.00012227806371455693 trainer/global_step‚4849 % _timestamp‚1.7844636852864125e+09 _runtime‚3359.674787424 _step‚396\óêÿÎ£È ' lr-AdamW/pg1‚0.00012227806371455693 ' lr-AdamW/pg2‚0.00012227806371455693 trainer/global_step‚4849 % _timestamp‚1.7844636852864125e+09 _runtime‚3359.674787424 _step‚396ŒÄå †2¤- _wandbruntime‚3359  _runtime‚3359R‚•Ø¥ÜO WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 _step‚397 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚186 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚376452898323968 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚10994140 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚286074653582336 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚37645èxUv "2898323968 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4002758366720 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚380455656690688 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚87270220 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚376000855964523 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚38466223 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚36816097 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4002847026688 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4120821260800 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚380455762390016 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4002847026688 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚25821957 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚36816287 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚10994326 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚281953715460608 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3991869544533 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 trainer/global_step‚4849 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚376452915363328 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 210970624 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚48803997 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚48793854 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚48793854 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4002758366720 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚38466221 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4120938121728 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3991782758107 _runtime‚3359.675596165 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚286074442611712 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 116860928 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚380455762390016 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚281953621350912 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚380455656690688 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚25821961 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 150006272 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4120938121728 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 3991782758107 _runtime‚3359.675596165 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚286074442611712 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 116860928 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚380455762390016 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚281953621350912 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚380455656690688 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚25821961 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 150006272 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4002847026688 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4120821260800 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚380455762390016 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4002847026688 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚25821957 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚36816287 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚10994326 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚281953715460608 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 3991869544533 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 trainer/global_step‚4849 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚376452915363328 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 210970624 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚48803997É颽v§‚(:ë Å‚óÒ‹Íê¤) disk./.usagePercent‚76.06507515484105 disk.dm-0.in‚ 1982.2109375" disk.nvme0c0n1.in‚ 1852.4453125 disk.nvme0n1.in‚ 1852.4453125& disk.nvme1n1.out‚16141.19873046875$ memory_percent‚99.20768737792969 disk.rbd0.out‚0& disk.nvme0n1.out‚34375.00244140625 disk.nvme1n1.in‚ 129.765625$ proc.memory.availableMB‚129.8125* proc.memory.percent‚17.773032188415527 cpu‚25.910757311160108 proc.cpu.threads‚17( disk.nvme0c0n1.out‚34375.00244140625 disk.md0.out‚ 82.3359375# proc.memory.rssMB‚ 2911.93359375% disk./.usageGB‚1167.7890663146973# disk.dm-0.out‚48344.88232421875 disk.dm-1.in‚0" disk.dm-1.out‚1993.95361328125 disk.rbd0.in‚0 disk.nvme1c1n1.in‚ 129.765625( disk.nvme1c1n1.out‚16141.19873046875 disk.md0.in‚0 network.sent‚ 7.192428e+06 network.recv‚8893380Í+“¨‚(:ˆ Å‚óÒáËš¥ gpu.0.gpu‚52.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚49.0 gpu.0.powerWatts‚222.078' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚74.026 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚06•)2©- _wandbruntime‚3360  _runtime‚3360“ìP}ì&ªæM M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4003632749568 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚181 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 trainer/global_step‚4849 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚87287683 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚38473852 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 _runtime‚3360.075472816 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚36823554 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚87277524 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚286131388500480 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 % _timestamp‚1.7844636856869357e+09 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚376528065029632 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4003544082432 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚25827013 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚286131603658240 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚282009759078912 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚380531697779200 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4121750469632 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚372083363731808 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚48803674 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚38473850 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚376528065029632 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚372083380771168 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4003632749568 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3992653039245 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚87277524 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚38473852 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚48813831 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚380531697779200 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚376528047990272 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚380531592072704 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚36823373 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚87287683 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚25827017 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4121629421568 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚282009853188608 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 121048064 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚376075929984515 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 150006272 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3992566252707 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚10996360 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812ÏcM· «ëM WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚380531697779200 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4121750469632 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚372083363731808 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚48803674 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚38473850 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚376528065029632 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚372083380771168 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4003632749568 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 3992653039245 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚87277524 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚38473852 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚48813831 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚380531697779200 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 WCDeviceStatsMonitor.on_train_ba+oJÑ:tch_end/active_bytes.large_pool.freed‚376528047990272 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚380531592072704 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚36823373 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 EDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚87287683 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚25827017 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4121629421568 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚282009853188608 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 121048064 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚376075929984515 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 150006272 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 3992566252707 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚10996360 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4003632749568 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚181 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 trainer/global_step‚4849 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚87287683 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚38473852 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 _runtime‚3360.075472816 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚36823554 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚87277524 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚286131388500480 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 % _timestamp‚1.7844636856869357e+09 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚376528065029632 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4003544082432 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚25827013 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚286131603658240 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚282009759078912 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760Žª ¿ö2¬- _wandbruntime‚3360  _runtime‚3360Ân¿r­þ -loss/FRt1/assignment_loss‚2.5022861104333 1loss/SRqqt1/detection_loss‚1.0686686020164626 0loss/FBt1/detection_loss‚0.47132569271244257 trainer/global_step‚4849 % _timestamp‚1.7844636856884854e+09 /loss/FRt2/assignment_loss‚1.035076802191527 1loss/SRqqt1/assignment_loss‚0.751037128502503 2loss/FBt1/assignment_loss‚0.045684865402785546 /loss/FRt1/detection_loss‚0.5518048637739943 /loss/FRt2/detection_loss‚1.4879929120916457 &loss/total_loss‚10.496696387276097 epoch‚12 1loss/FBt2/assignment_loss‚0.16067075545586235 /loss/SRqqt2/detection_loss‚0.63389374504222 /loss/FBt2/detection_loss‚1.1412031828210905 _runtime‚3360.076579508 _step‚399 2loss/SRqqt2/assignment_loss‚0.6470517268322636 Zè ®ƒ 2loss/SRqqt2/assignment_loss‚0.6470517268322636 -loss/FRt1/assignment_loss‚2.5022861104333 1loss/SRqqt1/detection_loss‚1.0686686020164626 0loss/FBt1/detection_loss‚0.47132569271244257 trainer/global_step‚4849 % _timestamp‚1.7844636856884854e+09 /loss/FRt2/assignment_loss‚1.035076802191527 1loss/SRqqt1/assignment_loss‚0.751037128502503 2loss/FBt1/assignment_loss‚0.045684865402785546 /loss/FRt1/detection_loss‚0.5518048637739943 /loss/FRt2/detection_loss‚1.4879929120916457 &loss/total_loss‚10.496696387276097 epoch‚12 1loss/FBt2/assignment_loss‚0.16067075545586235 /loss/SRqqt2/detection_loss‚0.63389374504222 /loss/FBt2/detection_loss‚1.1412031828210905 _runtime‚3360.076579508 _step‚399*Û"q¯‚(:æ Ô‚óÒÏŸå¤) proc.memory.percent‚17.77341365814209% disk./.usageGB‚1167.7889976501465' disk.nvme1c1n1.out‚16142.1513671875! disk.dm-1.out‚1994.1806640625 disk.rbd0.in‚0 disk.rbd0.out‚0( disk.nvme0c0n1.out‚34376.32177734375$ memory_percent‚99.20790195465088 cpu‚25.91189086547952 disk.nvme0n1.in‚ 1852.4453125 disk.nvme1n1.in‚ 129.765625 disk.md0.out‚82.875 network.recv‚893396) disk./.usagePercent‚76.06507068230884& disk.nvme0n1.out‚34376.32177734375% disk.nvme1n1.out‚16142.1513671875 disk.md0.in‚0 disk.dm-0.in‚ 1982.2109375# proc.memory.rssMB‚ 2911.99609375 proc.cpu.threads‚17 network.sent‚ 7.243536e+06 disk.nvme1c1n1.in‚ 129.765625# disk.dm-0.out‚48345.78857421875 disk.dm-1.in‚0" disk.nvme0c0n1.in‚ 1852.4453125( proc.memory.availableMB‚ 129.77734375 }“°‚(:ˆ Ô‚óÒñ×Á¥ gpu.0.gpu‚91.0 gpu.0.memory‚19, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚50.0 gpu.0.powerWatts‚186.834' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚62.278 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¡Rtn±‚(:ã ã‚óÒÎÇÞ¤ disk.rbd0.in‚0 disk.rbd0.out‚0" disk.nvme0c0n1.in‚ 1852.4453125( disk.nvme0c0n1.out‚34378.91552734375 network.sent‚ 7.297641e+06 disk.md0.in‚0 disk.dm-1.in‚0 disk.nvme0n1.in‚ 1852.4453125& disk.nvme0n1.out‚34378.91552734375 disk.nvme1c1n1.in‚ 129.765625$ memory_percent‚99.20806884765625# proc.memory.rssMB‚ 2911.99609375$ disk./.usageGB‚1167.788974761963# disk.dm-0.out‚48348.36376953125$ disk.nvme1n1.out‚16142.783203125" proc.memory.availableMB‚129.75) proc.memory.percent‚17.77341365814209 cpu‚25.912573805887707) disk./.usagePercent‚76.06506919146477& disk.nvme1c1n1.out‚16142.783203125 disk.nvme1n1.in‚ 129.765625 proc.cpu.threads‚17 network.recv‚897535 disk.md0.out‚ 83.15234375 disk.dm-0.in‚ 1982.2109375! disk.dm-1.out‚1994.2236328125"DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4031021288915 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚384252489946112 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4161532020736 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4161408868352 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚375723616515376 trainer/global_step‚4899 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚284760507750912 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚380210381639168 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚384252489946112 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚384252595645440 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚11103321 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚379754620764931 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4042108306944 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚380210381639168 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4042196966912 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 _runtime‚ 3392.48702512 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚191 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 217262080 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚379754724590717 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0ðÿ„æ'¸àO TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4031021288915 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚384252489946112 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4161532020736 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4161408868352 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚375723616515376 trainer/global_step‚4899 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚284760507750912 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚380210381639168 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚384252489946112 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚384252595645440 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚11103321 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚379754620764931 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4042108306944 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚380210381639168 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4042196966912 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 _runtime‚ 3392.48702512 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚191 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 217262080 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚379754724590717 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4031108075341 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚88134377 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚49286706 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚26074757 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚11103130 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚38847671 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚38847673 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚195 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚37177887 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚288921916619264 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚88144522 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4042196966912 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚49296849 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚26074761 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 123152384 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚49286706 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚380210398678528 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4042108306944 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚284760601860608 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚49296849 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚37178082 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 _step‚401 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 150006272 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚375723599476016 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚380210398678528 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 IDeviceStatsMon`¹A&Ó&itor.on_train_batch_end/inactive_split.all.freed‚37185136 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4042982689792 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚38855300 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚49296526 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚380285548344832 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 150006272 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚11105506 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚384328531034624 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚288979083490304 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚49306683 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚380285531305472 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚49306683 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚88161985 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚88151826 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚88151826 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚375797907040768 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚11105323 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚26079817 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4162343901696 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 trainer/global_step‚4899 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 221449216 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚187 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚379829694784923 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚284816645478912 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4162216562176 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚379829798610821 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚38855302 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚38855300 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚183 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚375797890001408 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4042894022656 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚88161985 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚288978862041088 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 $ _timestamp‚1.784463718517336e+09 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚380285531305472 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚37185323 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4031891570053 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4031804783515 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚384328425328128 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 è;‹ð&»êM FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚379829694784923 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚284816645478912 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4162216562176 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚379829798610821 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚38855302 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚38855300 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚183 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚375797890001408 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4042894022656 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚88161985 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚288978862041088 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 $ _timestamp‚1.784463718517336e+09 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚380285531305472 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚37185323 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4031891570053 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4031804783515 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚384328425328128 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚37185136 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4042982689792 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚38855300 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚49296526 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚380285548344832 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 150006272 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚11105506 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚384328531034624 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚288979083490304 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚49306683 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚380285531305472 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚49306683 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚88161985 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚88151826 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚88151826 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚375797907040768 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚11105323 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚26079817 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4162343901696 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 trainer/global_step‚4899 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 221449216 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚187 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048’–TM2¼- _wandbruntime‚3392  _runtime‚3392‚¼çï ½„ /loss/FRt2/detection_loss‚1.5161265172743028 /loss/FBt2/detection_loss‚1.9484561432618648 epoch‚12 0loss/FRt1/assignment_loss‚2.5505678490263777 0loss/FRt2/assignment_loss‚1.0900203231370673 2loss/SRqqt1/assignment_loss‚0.6488371398660724 2loss/FBt1/assignment_loss‚0.053725995323754244 2loss/FBt2/assignment_loss‚0.010881500042366099 /loss/FRt1/detection_loss‚0.5661533908264914 0loss/SRqqt1/detection_loss‚1.057636732551123 1loss/SRqqt2/detection_loss‚0.6535171712421062 2loss/SRqqt2/assignment_loss‚0.6258363820952842 /loss/FBt1/detection_loss‚0.7106006777579578 &loss/total_loss‚11.432359822404766 trainer/global_step‚4899 % _timestamp‚1.7844637185191903e+09 _runtime‚3392.907439568 _step‚403vm9%¾‰ 0loss/FRt1/assignment_loss‚2.5505678490263777 0loss/FRt2/assignment_loss‚1.0900203231370673 2loss/SRqqt1/assignment_loss‚0.6488371398660724 2loss/FBt1/assignment_loss‚0.053725995323754244 2loss/FBt2/assignment_loss‚0.010881500042366099 /loss/FRt1/detection_loss‚0.5661533908264914 0loss/SRqqt1/detection_loss‚1.057636732551123 1loss/SRqqt2/detection_loss‚0.6535171712421062 2loss/SRqqt2/assignment_loss‚0.6258363820952842 /loss/FBt1/detection_loss‚0.7106006777579578 &loss/total_loss‚11.432359822404766 trainer/global_step‚4899 % _timestamp‚1.7844637185191903e+09 _runtime‚3392.907439568 _step‚403 /loss/FRt2/detection_loss‚1.5161265172743028 /loss/FBt2/detection_loss‚1.9484561432618648 epoch‚12“*±^3r¿‚(:ç ò‚óÒé–ò¤ disk.md0.out‚ 83.36328125& disk.nvme0n1.out‚34379.91748046875$ memory_percent‚99.20847415924072( proc.memory.availableMB‚ 129.68359375) disk./.usagePercent‚76.06511814084509# disk.nvme1n1.out‚16143.31640625 disk.md0.in‚0# disk.dm-0.out‚48349.25927734375 disk.dm-1.in‚0 disk.rbd0.in‚0( disk.nvme0c0n1.out‚34379.91748046875 disk.nvme1c1n1.in‚ 129.765625" disk.nvme0c0n1.in‚ 1852.4453125) proc.memory.percent‚17.77341365814209 cpu‚25.912808735188914 network.sent‚ 7.300807e+06% disk.nvme1c1n1.out‚16143.31640625 disk.dm-0.in‚ 1982.2109375 disk.dm-1.out‚1994.392578125 disk.rbd0.out‚0 disk.nvme0n1.in‚ 1852.4453125# proc.memory.rssMB‚ 2911.99609375 proc.cpu.threads‚17 network.recv‚901028% disk./.usageGB‚1167.7897262573242 disk.nvme1n1.in‚ 129.765625 AÆ‘œÀ‚(:‘ ò‚óÒôèË¥ gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚49.0 gpu.0.powerWatts‚196.771' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚65.59033333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ø}ßmÁ‚(:â ƒóÒÊýؤ disk.rbd0.out‚0" disk.nvme0c0n1.in‚ 1852.4453125# disk.nvme0n1.out‚36671.92578125 disk.nvme1c1n1.in‚ 129.765625( disk.nvme1c1n1.out‚16249.90380859375 disk.nvme1n1.in‚ 129.765625 disk.md0.in‚0# disk.dm-0.out‚50742.96923828125 network.sent‚ 7.3642e+06 disk.rbd0.in‚0& disk.nvme1n1.out‚16249.90380859375 disk.md0.out‚ 83.8984375 disk.dm-1.in‚0 disk.dm-1.out‚1998.158203125( proc.memory.availableMB‚ 129.64453125# proc.memory.rssMB‚ 2911.99609375 disk.nvme0n1.in‚ 1852.4453125) proc.memory.percent‚17.77341365814209 cpu‚25.913119966688967 network.recv‚905018) disk./.usagePercent‚76.25281789407262# disk./.usageGB‚1170.67138671875% disk.nvme0c0n1.out‚36671.92578125 disk.dm-0.in‚ 1982.2109375$ memory_percent‚99.20871257781982 proc.cpu.threads‚17ÆTõ‚(:’ ƒóÒÕî—¥ gpu.0.gpu‚34.0 gpu.0.memory‚5, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚49.0 gpu.0.powerWatts‚219.466' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚73.15533333333335 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Y´Ó2Ã- _wandbruntime‚3426  _runtime‚3426ÔР…ÉÄà ' lr-AdamW/pg1‚0.00012154130393730402 ' lr-AdamW/pg2‚0.00012154130393730402 trainer/global_step‚4949 % _timestamp‚1.7844637522159662e+09 _runtime‚3426.604297169 _step‚404ôÈ<ëÎÅÈ ' lr-AdamW/pg2‚0.00012154130393730402 trainer/global_step‚4949 % _timestamp‚1.7844637522159662e+09 _runtime‚3426.604297169 _step‚404 ' lr-AdamW/pg1‚0.00012154130393730402”Å-¸ 2Æ- _wandbruntime‚3426  _runtime‚3426 $Œþ}ÇÛO _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4201940019200 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚388049428900864 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚39229123 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚383967864954368 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚287567394150912 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚49789701 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚189 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚379438125745616 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚26327557 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚11211017 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚26327561 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4070259819723 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚388049323201536 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚185 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4070346606149 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 G?_ÛFµdDeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4202067365888 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚388049428900864 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚39229123 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚383967864954368 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚287567394150912 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚49789701 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚189 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚379438125745616 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚26327557 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚11211017 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚26327561 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4070259819723 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚388049323201536 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚185 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4070346606149 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4202067365888 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚37545834 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4202742388224 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 133630976 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4082332630016 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 trainer/global_step‚4949 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚388125364290048 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4082243962880 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚388125364290048 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚11213406 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚287623531878912 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚384043031660032 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚89036287 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚49789378 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚49789378 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 _runtime‚3427.033718214 _step‚406 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚26332613 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚383583563411229 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚39236750 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚49799535 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚39236752 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4071043314323 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚89026128 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚379512433310368 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4071130100861 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚37546023 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚384043014620672 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚11213221 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚39236752 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚89036287 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚383583459585331 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚39236752 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚89036287 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚383583459585331 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚37545834 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4202742388224 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 133630976 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4082332630016 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 trainer/global_step‚4949 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚388125364290048 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4082243962880 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚388125364290048 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚11213406 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚287623531878912 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚384043031660032 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚89036287 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚49789378 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚49789378 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 _runtime‚3427.033718214 _step‚406 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚26332613 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚383583563411229 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚39236750 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚49799535 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812–lþz2Ì- _wandbruntime‚3427  _runtime‚3427Îäù¹HÍ€ /loss/FBt1/detection_loss‚0.5938478387012714 trainer/global_step‚4949 _runtime‚3427.034397505 1loss/FBt2/assignment_loss‚0.07519733582133893 &loss/total_loss‚10.257959308525301 .loss/FRt1/assignment_loss‚2.32311520745542 1loss/FBt1/assignment_loss‚0.06810063397937094 .loss/FRt1/detection_sªP¾loss‚0.528510340774264 /loss/FRt2/detection_loss‚1.4843176621183996 1loss/SRqqt1/detection_loss‚1.0192760316098566 /loss/FBt2/detection_loss‚0.9459493527652107 epoch‚12 % _timestamp‚1.7844637526454303e+09 0loss/FRt2/assignment_loss‚1.1319068368245453 2loss/SRqqt1/assignment_loss‚0.7033509954479735 2loss/SRqqt2/assignment_loss‚0.6446873553995004 1loss/SRqqt2/detection_loss‚0.7396997176281487 _step‚407ˆº9ˆ Î… /loss/FBt2/detection_loss‚0.9459493527652107 epoch‚12 % _timestamp‚1.7844637526454303e+09 0loss/FRt2/assignment_loss‚1.1319068368245453 2loss/SRqqt1/assignment_loss‚0.7033509954479735 2loss/SRqqt2/assignment_loss‚0.6446873553995004 1loss/SRqqt2/detection_loss‚0.7396997176281487 _step‚407 /loss/FBt1/detection_loss‚0.5938478387012714 trainer/global_step‚4949 _runtime‚3427.034397505 1loss/FBt2/assignment_loss‚0.07519733582133893 &loss/total_loss‚10.257959308525301 .loss/FRt1/assignment_loss‚2.32311520745542 1loss/FBt1/assignment_loss‚0.06810063397937094 .loss/FRt1/detection_loss‚0.528510340774264 /loss/FRt2/detection_loss‚1.4843176621183996 1loss/SRqqt1/detection_loss‚1.0192760316098566—¿J‹¢žÏ‚(:“ ƒóÒÖÜƤ gpu.0.gpu‚97.0 gpu.0.memory‚23, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚48.0 gpu.0.powerWatts‚114.427' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚38.14233333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0&÷noЂ(:ä ƒóÒ÷¹¹¥' disk.nvme0c0n1.out‚37820.1748046875% disk.nvme0n1.out‚37820.1748046875# proc.memory.rssMB‚ 2911.99609375) disk./.usagePercent‚76.34710533342111 disk.md0.out‚84.1875" disk.dm-0.out‚52221.6533203125" disk.nvme0c0n1.in‚ 1852.4453125$ memory_percent‚99.20916557312012' proc.memory.availableMB‚ 129.5703125) proc.memory.percent‚17.77341365814209 network.recv‚908496% disk./.usageGB‚1172.1189346313477 disk.md0.in‚0! disk.dm-1.out‚1998.3505859375 disk.rbd0.in‚0 disk.rbd0.out‚0 disk.nvme0n1.in‚ 1852.4453125 cpu‚25.913340979058304 network.sent‚ 7.381859e+06 disk.dm-0.in‚ 1982.2109375 disk.dm-1.in‚0 disk.nvme1c1n1.in‚ 129.765625' disk.nvme1c1n1.out‚16581.1611328125 disk.nvme1n1.in‚ 129.765625% disk.nvme1n1.out‚16581.1611328125 proc.cpu.threads‚17_ÁºfÑ‚(:Û ŸƒóÒü×Ö¤# proc.memory.rssMB‚ 2911.99609375 proc.cpu.threads‚17 network.recv‚912459$ disk./.usageGB‚1172.122127532959 disk.md0.out‚ 84.453125 disk.dm-0.in‚ 1982.2109375" disk.dm-0.out‚52814.2685546875! disk.dm-1.out‚1998.5615234375 disk.rbd0.in‚0 disk.nvme0n1.in‚ 1852.4453125 disk.dm-1.in‚0 disk.rbd0.out‚0" disk.nvme0c0n1.in‚ 1852.4453125) proc.memory.percent‚17.77341365814209 cpu‚25.91481114858682 network.sent‚ 7.430926e+06) disk./.usagePercent‚76.34731330616896 disk.nvme1c1n1.in‚ 129.765625& disk.nvme1c1n1.out‚16584.849609375$ disk.nvme0n1.out‚38409.876953125" proc.memory.availableMB‚65.125 disk.nvme1n1.in‚ 129.765625$ disk.nvme1n1.out‚16584.849609375 disk.md0.in‚0& disk.nvme0c0n1.out‚38409.876953125$ memory_percent‚99.60250854492188/˜Êd‘Ò‚(:† ŸƒóÒ„¶›¥ gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚49.0 gpu.0.powerWatts‚223.554' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚74.518 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0«šB2Ó- _wandbruntime‚3461  _runtime‚3461ìaþ9ÈÔ $ _timestamp‚1.784463787032728e+09 _runtime‚3461.421136341 _step‚408 ' lr-AdamW/pg1‚0.00012116791007687333 ' lr-AdamW/pg2‚0.00012116791007687333 trainer/global_step‚4999Ÿ46ZÍÕÇ ' lr-AdamW/pg1‚0.00012116791007687333 ' lr-AdamW/pg2‚0.00012116791007687333 trainer/global_step‚4999 $ _timestamp‚1.784463787032728e+09 _runtime‚3461.421136341 _step‚408˜7²2Ö- _wandbruntime‚3461  _runtime‚3461·×^÷â'×ÜO N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 _step‚409 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4120896847360 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚37899713 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚387262150365747 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚191 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 trainer/global_step‚4999 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚11319352 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4242562366464 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚50272410 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4120808187392 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚50282553 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚39610573 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚387262254191533 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚11319165 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚383152669054576 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4120896847360 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚187 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 223553536 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4109585136957 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚26580357 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚89882981 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚387725365308928 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚89893126 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚290374374660608 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 129443840 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚387725365308928 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚383152652015216 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚37899522 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚290374280550912 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4109498350531 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚39610571 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚391846262156288 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚294616937027072 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4120808187392 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚26580361 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚387725348269568 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚39610573 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚50272410 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4242432922624 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚50282553 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚387725348269568 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚294616713473536 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚391846156456960 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚391846262156288 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960v„Èç'ØáO cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4242562366464 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚50272410 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4120808187392 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚50282553 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚39610573 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚387262254191533 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚11319165 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚383152669054576 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4120896847360 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚187 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 223553536 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4109585136957 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚26580357 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚89882981 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚387725365308928 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚89893126 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚290374374660608 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 129443840 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚387725365308928 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚383152652015216 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚37899522 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚290374280550912 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4109498350531 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚39610571 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚391846262156288 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚294616937027072 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4120808187392 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚26580361 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚387725348269568 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚39610573 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚50272410 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4242432922624 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚50282553 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚387725348269568 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚294616713473536 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚391846156456960 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚391846262156288 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 _step‚409 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4120896847360 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚37899713 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚387262150365747 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚191 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 trainer/global_step‚4999 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚11319352™Ò¯?&2Ù- _wandbruntime‚3461  _runtime‚3461˜|z(ÝÚæM J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚50282230 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚391922197545472 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚37906658 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚195 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚26585417 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚383226959579968 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚391922197545472 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 133630976 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 trainer/global_step‚4999 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4121682570240 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4121593903104 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚290430512388608 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚387800497935872 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚391922091838976 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚50292387 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚387800514975232 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚89910589 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4110281845131 _runtime‚3461.841704659 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚294673664080896 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚50282230 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 227740672 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚290430418278912 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4110368631669 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚39618200 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚387800514975232 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚294673891821568 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚39618202 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 150006272 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ;5DeviceStatsMonitor.on_train_batch_e.Q»nd/num_device_free‚0 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚89910589 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚39618200 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4243379432960 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚37906857 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚50292387þ<ñ&ÛëM WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚387800497935872 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚391922091838976 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚50292387 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚387800514975232 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚89910589 B6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4110281845131 _runtime‚3461.841704659 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚294673664080896 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚50282230 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 227740672 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚290430418278912 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4110368631669 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚39618200 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚387800514975232 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚294673891821568 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚39618202 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 150006272 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚89910589 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚39618200 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4243379432960 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚37906857 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚50292387 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚50282230 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚391922197545472 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚37906658 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚195 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚26585417 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚383226959579968 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚391922197545472 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 133630976 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 trainer/global_step‚4999 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4121682570240 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4121593903104 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚290430512388608 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0šN6”ù2Ü- _wandbruntime‚3461  _runtime‚3461ÆHRGÝü &loss/total_loss‚10.411619785419365 1loss/SRqqt2/assignment_loss‚0.658496668537213 1loss/FBt1/assignment_loss‚0.04600475393159974 /loss/FRt1/detection_loss‚0.5547121105986709 /loss/FRt2/detection_loss‚1.4174397195159638 _runtime‚3461.842757192 0loss/FRt1/assignment_loss‚2.4503949313456643 /loss/FRt2/assignment_loss‚1.013017825139772 2loss/SRqqt1/assignment_loss‚0.6813798249386886 2loss/FBt2/assignment_loss‚0.021122299594399548 -loss/FBt1/detection_loss‚0.62404435102041 +loss/FBt2/detection_loss‚1.268322016147 % _timestamp‚1.7844637874542315e+09 1loss/SRqqt1/detection_loss‚1.0383073255578648 1loss/SRqqt2/detection_loss‚0.6383779590921195 epoch‚12 trainer/global_step‚4999 _step‚411í' +Þ trainer/global_step‚4999 _step‚411 &loss/total_loss‚10.411619785419365 1loss/SRqqt2/assignment_loss‚0.658496668537213 1loss/FBt1/assignment_loss‚0.04600475393159974 /loss/FRt1/detection_loss‚0.5547121105986709 /loss/FRt2/detection_loss‚1.4174397195159638 _runtime‚3461.842757192 0loss/FRt1/assignment_loss‚2.4503949313456643 /loss/FRt2/assignment_loss‚1.013017825139772 2loss/SRqqt1/assignment_loss‚0.6813798249386886 2loss/FBt2/assignment_loss‚0.021122299594399548 -loss/FBt1/detection_loss‚0.62404435102041 +loss/FBt2/detection_loss‚1.268322016147 % _timestamp‚1.7844637874542315e+09 1loss/SRqqt1/detection_loss‚1.0383073255578648 1loss/SRqqt2/detection_loss‚0.6383779590921195 epoch‚12›žŽ}ˆlß‚(:á ®ƒóÒÙ¤‹¥ disk.md0.out‚ 84.828125 disk.dm-1.in‚0 disk.rbd0.in‚0% disk.nvme1n1.out‚16585.6474609375# proc.memory.rssMB‚ 2911.99609375 disk.dm-0.in‚ 1982.2109375" disk.dm-0.out‚52815.4912109375" disk.nvme0c0n1.in‚ 1852.4453125% disk.nvme0n1.out‚38411.3740234375' disk.nvme1c1n1.out‚16585.6474609375 disk.nvme1n1.in‚ 129.765625$ proc.memory.availableMB‚64.84375 proc.cpu.threads‚17 disk.rbd0.out‚0 network.sent‚ 7.44914e+06) disk./.usagePercent‚76.34731131837687 disk.md0.in‚0 disk.nvme0n1.in‚ 1852.4453125 disk.nvme1c1n1.in‚ 129.765625# memory_percent‚99.6042251586914) proc.memory.percent‚17.77341365814209! disk.dm-1.out‚1998.8193359375 cpu‚25.915028362834196' disk.nvme0c0n1.out‚38411.3740234375 network.recv‚916085% disk./.usageGB‚1172.12209701538091Å×`žà‚(:“ ®ƒóÒîßÄ¥ gpu.0.gpu‚80.0 gpu.0.memory‚19, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚49.0 gpu.0.powerWatts‚223.925' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚74.64166666666668 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0EAÓœpá‚(:å ½ƒóÒ‹Žà¤# proc.memory.rssMB‚ 2911.99609375( disk./.usagePercent‚76.3472698232169 disk.md0.in‚0# disk.dm-0.out‚52821.56982421875 disk.dm-1.in‚0 disk.rbd0.out‚0" disk.nvme0c0n1.in‚ 1852.4453125 disk.nvme1n1.in‚ 129.765625 proc.cpu.threads‚17 network.sent‚ 7.497473e+06% disk./.usageGB‚1172.1214599609375( disk.nvme0c0n1.out‚38417.45458984375) proc.memory.percent‚17.77341365814209 cpu‚25.913466255860797 disk.md0.out‚85.125& disk.nvme0n1.out‚38417.45458984375 disk.nvme1c1n1.in‚ 129.765625% disk.nvme1n1.out‚16586.3876953125$ memory_percent‚99.60286617279053 network.recv‚919929 disk.dm-0.in‚ 1982.2109375! disk.dm-1.out‚1998.9013671875 disk.rbd0.in‚0 disk.nvme0n1.in‚ 1852.4453125' disk.nvme1c1n1.out‚16586.3876953125' proc.memory.availableMB‚ 65.06640625j²©‘â‚(:† ½ƒóҧ gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚47.0 gpu.0.powerWatts‚114.024' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚38.008 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0s•uã‚(:ê ̃óÒÜ”ñ¤" disk.dm-1.out‚1999.03076171875 disk.rbd0.out‚0" disk.nvme0c0n1.in‚ 1852.4453125 disk.nvme1n1.in‚ 129.765625 proc.cpu.threads‚17 network.recv‚923552) disk./.usagePercent‚76.34733691120007% disk./.usageGB‚1172.1224899291992# disk.dm-0.out‚52822.45263671875# proc.memory.rssMB‚ 2911.99609375 disk.md0.in‚0 disk.dm-1.in‚0( disk.nvme0c0n1.out‚38418.62939453125& disk.nvme0n1.out‚38418.62939453125 disk.nvme1c1n1.in‚ 129.765625( disk.nvme1c1n1.out‚16587.11376953125# proc.memory.availableMB‚65.0625 network.sent‚ 7.503529e+06 disk.md0.out‚ 85.53515625 disk.rbd0.in‚0 disk.nvme0n1.in‚ 1852.4453125& disk.nvme1n1.out‚16587.11376953125$ memory_percent‚99.60289001464844) proc.memory.percent‚17.77341365814209 cpu‚25.911910221451397 disk.dm-0.in‚ 1982.2109375þGÆ>œä‚(:‘ ̃óҪ೥ gpu.0.gpu‚34.0 gpu.0.memory‚8, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚49.0 gpu.0.powerWatts‚217.51' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚72.50333333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0$ª—'2å- _wandbruntime‚3495  _runtime‚3495‡äO“Éæà _step‚412 ' lr-AdamW/pg1‚0.00012079120547369865 ' lr-AdamW/pg2‚0.00012079120547369865 trainer/global_step‚5049 % _timestamp‚1.7844638212912045e+09 _runtime‚3495.679534334Á9ÚÎçÈ trainer/global_step‚5049 % _timestamp‚1.7844638212912045e+09 _runtime‚3495.679534334 _step‚412 ' lr-AdamW/pg1‚0.00012079120547369865 ' lr-AdamW/pg2‚0.00012079120547369865œ¦^ÛY2è- _wandbruntime‚3495  _runtime‚3495‘š«â'éÜO UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚11425389 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _runtime‚3495.680282646 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚26833157 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚395642989712384 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚391482831584768 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚39992023 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚293181166950912 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚182 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 150006272 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚395643095411712 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚178 trainer/global_step‚5049 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚391015915166155 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚50775405 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚50765262 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 _step‚413 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚391482831584768 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4160158127616 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚26833161 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4148823667765ç±²ŽUêáO TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚50775405 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4148736881339 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚391482848624128 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚395643095411712 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚178 trainer/global_step‚5049 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚391015915166155 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚50775405 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚50765262 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 _step‚413 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚391482831584768 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4160158127616 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚26833161 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4148823667765 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚11425389 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 _runtime‚3495.680282646 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚26833157 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚395642989712384 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚391482831584768 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚39992023 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚293181166950912 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚182 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 150006272 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 I8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚26838217 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 150006272 BDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚391557998290432 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4160943843328 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚386941468810208 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚90784891 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚39999650 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚391557998290432 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4161032510464 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚50775082 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚90774732 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚50785239 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚391557981251072 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚293237398788608 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚38265785 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚181 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4283667132928 % _timestamp‚1.7844638217127624e+09 _step‚414 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚395718925094400 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚90774732 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚26838213 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚391091093012045 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚386941485849568 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚39999652 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚297521197455360 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4283798666752 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚391090989186147 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4149607162477 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚391557981251072 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4161032510464 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚90784891 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 225643520 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4149520375939 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚39999652 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚50785239 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 _runtime‚ 3496.1013034 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚395719030800896œõtfï&íéM PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚391557998290432 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4160943843328 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚386941468810208 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚90784891 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚39999650 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚391557998290432 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4161032510464 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚50775082 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚90774732 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚50785239 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚391557981251072 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚293237398788608 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚38265785 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚181 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4283667132928 % _timestamp‚1.7844638217127624e+09 _step‚414 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚395718925094400 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚90774732 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚26838213 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚391091093012045 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚386941485849568 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚39999652 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚297521197455360 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4283798666752 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚391090989186147 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4149607162477 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚391557981251072 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4161032510464 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚90784891 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 225643520 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4149520375939 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚39999652 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚50785239 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 _runtime‚ 3496.1013034 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚395719030800896 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚185 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚26838217 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 150006272 B¬Iú2î- _wandbruntime‚3496  _runtime‚3496Û2ï /loss/FBt1/detection_loss‚0.5006608044126885 epoch‚12 0loss/FRt1/assignment_loss‚2.3526796191869574 2loss/SRqqt1/assignment_loss‚0.6433426478426323 1loss/SRqqt2/detection_loss‚0.6809568541451853 1loss/SRqqt2/assignment_loss‚0.590513187756135 1loss/FBt1/assignment_loss‚0.07011011468040751 2loss/FBt2/assignment_loss‚0.004601492867961731 /loss/FRt2/detection_loss‚1.5018335841395105 /loss/FBt2/detection_loss‚1.4574060074433133 trainer/global_step‚5049 _runtime‚3496.102328801 _step‚415 0loss/FRt2/assignment_loss‚1.0141495050017701 /loss/FRt1/detection_loss‚0.5195007933920766 1loss/SRqqt1/detection_loss‚1.0411901786398616 $loss/total_loss‚10.3769447895085 % _timestamp‚1.7844638217138202e+09ŠŒô ð† /loss/FRt2/detection_loss‚1.5018335841395105 /loss/FBt2/detection_loss‚1.4574060074433133 trainer/global_step‚5049 _runtime‚3496.102328801 _step‚415 0loss/FRt2/assignment_loss‚1.0141495050017701 /loss/FRt1/detection_loss‚0.5195007933920766 1loss/SRqqt1/detection_loss‚1.0411901786398616 $loss/total_loss‚10.3769447895085 % _timestamp‚1.7844638217138202e+09 /loss/FBt1/detection_loss‚0.5006608044126885 epoch‚12 0loss/FRt1/assignment_loss‚2.3526796191869574 2loss/SRqqt1/assignment_loss‚0.6433426478426323 1loss/SRqqt2/detection_loss‚0.6809568541451853 1loss/SRqqt2/assignment_loss‚0.590513187756135 1loss/FBt1/assignment_loss‚0.07011011468040751 2loss/FBt2/assignment_loss‚0.004601492867961731ŸP» \mñ‚(:â ÛƒóÒ“Ž´¥ disk.dm-0.in‚ 1982.2109375" disk.nvme0n1.out‚ 38421.6640625 disk.nvme1c1n1.in‚ 129.765625) proc.memory.percent‚17.77341365814209 cpu‚25.910432403127103$ disk.nvme0c0n1.out‚ 38421.6640625 disk.nvme0n1.in‚ 1852.4453125 disk.dm-1.in‚0" disk.dm-1.out‚1999.13623046875" disk.nvme0c0n1.in‚ 1852.4453125 disk.nvme1n1.in‚ 129.765625& disk.nvme1n1.out‚16588.01513671875 disk.md0.out‚ 86.00390625% disk./.usageGB‚1172.1223526000977 disk.dm-0.out‚52825.27734375& proc.memory.availableMB‚ 64.9921875# proc.memory.rssMB‚ 2911.99609375 proc.cpu.threads‚17 network.sent‚ 7.521185e+06 network.recv‚927014) disk./.usagePercent‚76.34732796613565 disk.rbd0.in‚0 disk.rbd0.out‚0( disk.nvme1c1n1.out‚16588.01513671875 disk.md0.in‚0$ memory_percent‚99.60331916809082à Æò‚(:’ ÛƒóÒ܇⥠gpu.0.gpu‚27.0 gpu.0.memory‚6, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚47.0 gpu.0.powerWatts‚114.007' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚38.00233333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¼HÇCpó‚(:å êƒóÒéÊܤ$ memory_percent‚99.60348606109619 proc.cpu.threads‚17" disk.nvme0c0n1.in‚ 1852.4453125 disk.nvme0n1.in‚ 1852.4453125 disk.nvme1c1n1.in‚ 129.765625( disk.nvme1c1n1.out‚16588.60693359375& disk.nvme1n1.out‚16588.60693359375 disk.dm-0.out‚ 52825.8359375 network.sent‚ 7.569581e+06) disk./.usagePercent‚76.34732150581135% disk.nvme0n1.out‚38422.3974609375 disk.md0.out‚ 86.296875 disk.dm-0.in‚ 1982.2109375' proc.memory.availableMB‚ 64.96484375# proc.memory.rssMB‚ 2911.99609375) proc.memory.percent‚17.77341365814209% disk./.usageGB‚1172.1222534179688 disk.rbd0.in‚0 disk.rbd0.out‚0' disk.nvme0c0n1.out‚38422.3974609375 disk.nvme1n1.in‚ 129.765625 disk.md0.in‚0 cpu‚25.90891498390377 network.recv‚931078 disk.dm-1.in‚0" disk.dm-1.out‚1999.256347656255Åi|žô‚(:“ êƒóÒùؤ¥ gpu.0.gpu‚55.0 gpu.0.memory‚13, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚49.0 gpu.0.powerWatts‚220.006' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚73.33533333333332 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚00[ZÎ2õ- _wandbruntime‚3529  _runtime‚3529šØ&Éöà ' lr-AdamW/pg1‚0.00012041121432281271 ' lr-AdamW/pg2‚0.00012041121432281271 trainer/global_step‚5099 % _timestamp‚1.7844638551283715e+09 _runtime‚3529.517811059 _step‚416jžhvÎ÷È % _timestamp‚1.7844638551283715e+09 _runtime‚3529.517811059 _step‚416 ' lr-AdamW/pg1‚0.00012041121432281271 ' lr-AdamW/pg2‚0.00012041121432281271 trainer/global_step‚5099 ²¯°2ø- _wandbruntime‚3529  _runtime‚3529ê¼å˜›ùÜO [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4199596727808 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4199508067840 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4323239182848 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4199508067840 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚38618195 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚51268257 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚399439928667136 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚40373473 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚f´ä¦G 4199596727808 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 238233600 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚395240314899968 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚11532238 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚27085961 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚390581721593776 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚390581704554416 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚51268257 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4323383306752 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚399439822967808 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚51258114 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚11532422 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚295988147460608 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 144123904 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 150006272 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚399439822967808 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚51258114 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚394769783792349 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚40373473 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚300311530767360 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚395240314899968 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚91641730 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚40373471 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚394769679966563 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚188 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 _runtime‚3529.517877549 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426uЖþç'úáO [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚38618195 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚51268257 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚399439928667136 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚40373473 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4199596727808 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 238233600 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚395240314899968 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚11532238 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚27085961 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚390581721593776 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚390581704554416 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚51268257 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4323383306752 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚399439822967808 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚51258114 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚11532422 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚295988147460608 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 144123904 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 150006272 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚399439822967808 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚51258114 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚394769783792349 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚40373473 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚300311530767360 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚395240314899968 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚91641730 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚40373471 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚394769679966563 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚188 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 _runtime‚3529.517877549 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4199596727808 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4199508067840 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4323239182848 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4199508067840 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚300368242338304 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚38625377 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚394844857812453 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚51267934 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 131533824 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚40381100 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚399515864056320 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 150006272 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚27091017 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚27091013 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 _step‚418 % _timestamp‚1.7844638555482292e+09 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4200382450688 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚296044191078912 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4200382450688 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4200293783552 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4324182793216 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4324051259392 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 _runtime‚3529.936758635 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚395315464566272 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚91659193 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚91659193 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚11534364 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚40381102 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚177 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚51278091 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚394844753986555 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4200293783552 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚40381100 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚181 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚395315464566272 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚395315481605632 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚395315481605632 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚399515864056320 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚390656012119168 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚91649034 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚91649034 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 225643520 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚296044285188608 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4188845693285 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4188758906747 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚38625558¯Ü‘ýëM UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4200293783552 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚40381100 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚181 BDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚395315464566272 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚395315481605632 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚395315481605632 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚399515864056320 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚390656012119168 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚91649034 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚91649034 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 225643520 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚296044285188608 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4188845693285 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4188758906747 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚38625558 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚300368467981824 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚390655995079808 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚40381102 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚51278091 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 trainer/global_step‚5099 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚300368242338304 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 FDeviceSta:<`tsMonitor.on_train_batch_end/inactive_split.all.freed‚38625377 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚394844857812453 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚51267934 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 131533824 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚40381100 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚399515864056320 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 150006272 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚27091017 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚27091013 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 _step‚418 % _timestamp‚1.7844638555482292e+09 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4200382450688 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚296044191078912 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4200382450688 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4200293783552 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4324182793216 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4324051259392 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 _runtime‚3529.936758635 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚395315464566272 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚91659193 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚91659193 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚11534364 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚40381102 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚177 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚51278091 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚394844753986555¢ZA$%2þ- _wandbruntime‚3529  _runtime‚3529~ìÿþ /loss/FRt1/assignment_loss‚2.358350604265294 2loss/SRqqt2/assignment_loss‚0.6046956784373447 1loss/FBt2/assignment_loss‚0.18139416430477012 &loss/total_loss‚10.331352810223905 trainer/global_step‚5099 2loss/SRqqt1/assignment_loss‚0.6298936550947241 0loss/FBt1/assignment_loss‚0.0573567462958491 .loss/FRt1/detection_loss‚0.587290714456724 % _timestamp‚1.7844638555492978e+09 _step‚419 0loss/FRt2/assignment_loss‚1.0865327478118643 /loss/FRt2/detection_loss‚1.4448056939770322 0loss/SRqqt1/detection_loss‚1.081220157784768 1loss/SRqqt2/detection_loss‚0.6664361769778565 _runtime‚3529.937824836 .loss/FBt1/detection_loss‚0.645882663218729 /loss/FBt2/detection_loss‚0.9874938075989484 epoch‚12¥³»_ €ƒ &loss/total_loss‚10.331352810223905 trainer/global_step‚5099 2loss/SRqqt1/assignment_loss‚0.6298936550947241 0loss/FBt1/assignment_loss‚0.0573567462958491 .loss/FRt1/detection_loss‚0.587290714456724 % _timestamp‚1.7844638555492978e+09 _step‚419 0loss/FRt2/assignment_loss‚1.0865327478118643 /loss/FRt2/detection_loss‚1.4448056939770322 0loss/SRqqt1/detection_loss‚1.081220157784768 1loss/SRqqt2/detection_loss‚0.6664361769778565 _runtime‚3529.937824836 .loss/FBt1/detection_loss‚0.645882663218729 /loss/FBt2/detection_loss‚0.9874938075989484 epoch‚12 /loss/FRt1/assignment_loss‚2.358350604265294 2loss/SRqqt2/assignment_loss‚0.6046956784373447 1loss/FBt2/assignment_loss‚0.18139416430477012£5Rox‚(:í ùƒóÒ˜À›¥# disk.dm-0.out‚52826.86669921875 disk.dm-1.in‚0$ memory_percent‚99.60391521453857 network.sent‚ 7.588343e+06 network.recv‚934596% disk./.usageGB‚1172.1223678588867& disk.nvme0n1.out‚38423.66845703125 disk.rbd0.out‚0( disk./.usagePercent‚76.3473289600317 disk.nvme0n1.in‚ 1852.4453125( disk.nvme1c1n1.out‚16589.16748046875 disk.md0.out‚ 86.61328125" disk.dm-1.out‚1999.36572265625 disk.rbd0.in‚0' proc.memory.availableMB‚ 64.89453125# proc.memory.rssMB‚ 2911.99609375( disk.nvme0c0n1.out‚38423.66845703125 disk.nvme1c1n1.in‚ 129.765625& disk.nvme1n1.out‚16589.16748046875 disk.dm-0.in‚ 1982.2109375) proc.memory.percent‚17.77341365814209 cpu‚25.907392371335302 proc.cpu.threads‚17" disk.nvme0c0n1.in‚ 1852.4453125 disk.nvme1n1.in‚ 129.765625 disk.md0.in‚0Ní‘‚‚(:† ùƒóÒž²É¥ gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚49.0 gpu.0.powerWatts‚114.099' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚38.033 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ûâ|Øoƒ‚(:ä ˆ„óÒ—²Ü¤) proc.memory.percent‚17.77341365814209 cpu‚25.90589759526472 network.sent‚ 7.638592e+06 network.recv‚938473% disk./.usageGB‚1172.1223602294922 disk.md0.out‚ 87.05078125 disk.rbd0.out‚0" disk.nvme0c0n1.in‚ 1852.4453125 disk.nvme1c1n1.in‚ 129.765625# disk.nvme1n1.out‚16589.86328125) disk./.usagePercent‚76.34732846308367# disk.dm-0.out‚52827.52294921875 disk.dm-1.in‚0( disk.nvme0c0n1.out‚38424.64306640625& disk.nvme0n1.out‚38424.64306640625% disk.nvme1c1n1.out‚16589.86328125 disk.nvme1n1.in‚ 129.765625% proc.memory.availableMB‚ 64.890625 disk.nvme0n1.in‚ 1852.4453125$ memory_percent‚99.60393905639648# proc.memory.rssMB‚ 2911.99609375 proc.cpu.threads‚17 disk.md0.in‚0 disk.dm-0.in‚ 1982.2109375! disk.dm-1.out‚1999.4462890625 disk.rbd0.in‚0™:?©ž„‚(:“ ˆ„óÒÅöŒ¥ gpu.0.gpu‚41.0 gpu.0.memory‚10, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚49.0 gpu.0.powerWatts‚224.144' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚74.71466666666667 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0:ÿ42…- _wandbruntime‚3563  _runtime‚3563Eò)˜É†Ã ' lr-AdamW/pg2‚0.00012002796103033705 trainer/global_step‚5149 % _timestamp‚1.7844638892971966e+09 _runtime‚3563.685580209 _step‚420 ' lr-AdamW/pg1‚0.00012002796103033705:â{lÎ‡È % _timestamp‚1.7844638892971966e+09 _runtime‚3563.685580209 _step‚420 ' lr-AdamW/pg1‚0.00012002796103033705 ' lr-AdamW/pg2‚0.00012002796103033705 trainer/global_step‚5149¤¸ LJ2ˆ- _wandbruntime‚3563  _runtime‚3563문uá'‰ÛO R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚403236656223232 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚51750966 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚27338761 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚398523444766971 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚38977464 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4363698874368 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚403236761922560 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚403236761922560 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚40754923 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 _runtime‚3563.689847696 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚51750966 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚11638535 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚398523548592757 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚398997798215168 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚298795033860608 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚27338757 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚303158868470272 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚40754923 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚40754921 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚394296247863376 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4227213942955 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚403236656223232 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚303158638625280 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4238946668032 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4238946668032 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚40754921 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚398997798215168 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚168 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚394296230824016 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4227300729381 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚403236656223232 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚51750966 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚27338761 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚398523444766971 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚38977464 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4363698874368 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚403236761922560 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚403236761922560 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚40754923 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 _runtime‚3563.689847696 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚51750966 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚11638535 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚398523548592757 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚398997798215168 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚298795033860608 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚27338757 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚303158868470272 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚40754923 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚40754921 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚394296247863376 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4227213942955 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚403236656223232 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚303158638625280 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4238946668032 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4238946668032 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0¥üOõ2‹- _wandbruntime‚3564  _runtime‚3564•¶³IçŒæM S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚403312591605248 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚399072964920832 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚172 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚40762550 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚92533495 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚11640746 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚40762552 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚176 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 trainer/global_step‚5149 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4239643723776 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4364636185088 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚399072947881472 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚403312697311744 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚38984387 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚298851077478912 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚92533495 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚398598518786963 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚399072964920832 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚399072947881472 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 150006272 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 % _timestamp‚1.7844638897181816e+09 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚303215580033024 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚40762552 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚27343813 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚51770943 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4364502554112 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4228084224093 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚51760786 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 133630976 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚394370521349408 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚92523336 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚40762550 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚27343817 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 _runtime‚3564.106697155 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚403312697311744 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚92533495 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚11640746 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚40762552 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚176 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 trainer/global_step‚5149 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4239643723776 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4364636185088 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚399072947881472 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚403312697311744 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚38984387 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚298851077478912 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚92533495 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚398598518786963 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚399072964920832 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚399072947881472 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 150006272 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 % _timestamp‚1.7844638897181816e+09 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚303215580033024 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚40762552 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚27343813 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚51770943 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4364502554112 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4228084224093 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚51760786 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 133630976 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚394370521349408 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚92523336 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚40762550 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚27343817 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0¦`Ö^ 2Ž- _wandbruntime‚3564  _runtime‚3564j[¨l ƒ 0loss/FRt1/assignment_loss‚2.3326677699926597 2loss/SRqqt2/assignment_loss‚0.6164478303600308 1loss/FBt1/assignment_loss‚0.09526143977494576 0loss/SRqqt1/detection_loss‚1.121257336282861 &loss/total_loss‚10.258934827037095 % _timestamp‚1.7844638897192204e+09 0loss/FRt2/assignment_loss‚0.9749284305105368 2loss/FBt2/assignment_loss‚0.035115059177302475 1loss/SRqqt2/detection_loss‚0.6678542967666277 /loss/FBt2/detection_loss‚1.1234518756037173 _runtime‚3564.110803902 epoch‚12 /loss/FRt1/detection_loss‚0.5629512074224922 /loss/FRt2/detection_loss‚1.4081969500589055 /loss/FBt1/detection_loss‚0.5816690796695639 trainer/global_step‚5149 _step‚423 2loss/SRqqt1/assignment_loss‚0.7391335514174509— Ĉ 0loss/FRt1/assignment_loss‚2.3326677699926597 2loss/SRqqt2/assignment_loss‚0.6164478303600308 1loss/FBt1/assignment_loss‚0.09526143977494576 0loss/SRqqt1/detection_loss‚1.121257336282861 &loss/total_loss‚10.258934827037095 % _timestamp‚1.7844638897192204e+09 0loss/FRt2/assignment_loss‚0.9749284305105368 2loss/FBt2/assignment_loss‚0.035115059177302475 1loss/SRqqt2/detection_loss‚0.6678542967666277 /loss/FBt2/detection_loss‚1.1234518756037173 _runtime‚3564.110803902 epoch‚12 /loss/FRt1/detection_loss‚0.5629512074224922 /loss/FRt2/detection_loss‚1.4081969500589055 /loss/FBt1/detection_loss‚0.5816690796695639 trainer/global_step‚5149 _step‚423 2loss/SRqqt1/assignment_loss‚0.7391335514174509§DO`Ôo‘‚(:ä —„óÒ¼ú¥) disk./.usagePercent‚76.34732722071362% disk./.usageGB‚1172.1223411560059 disk.dm-0.in‚ 1982.2109375 disk.dm-1.in‚0 disk.rbd0.out‚0" disk.nvme0c0n1.in‚ 1852.4453125 disk.nvme1c1n1.in‚ 129.765625 network.recv‚941678 disk.md0.in‚0 disk.rbd0.in‚0( disk.nvme0c0n1.out‚38426.67041015625% disk.nvme1c1n1.out‚16590.37109375# disk.nvme1n1.out‚16590.37109375& proc.memory.availableMB‚ 64.8359375# proc.memory.rssMB‚ 2911.99609375 disk.md0.out‚ 87.3046875! disk.dm-1.out‚1999.4931640625 cpu‚25.90454292912542 proc.cpu.threads‚17# disk.dm-0.out‚52829.47216796875 disk.nvme0n1.in‚ 1852.4453125& disk.nvme0n1.out‚38426.67041015625 disk.nvme1n1.in‚ 129.765625$ memory_percent‚99.60427284240723) proc.memory.percent‚17.77341365814209 network.sent‚ 7.641657e+067¨^Wž’‚(:“ —„óÒ°’¦ gpu.0.gpu‚98.0 gpu.0.memory‚23, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚49.0 gpu.0.powerWatts‚114.382' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚38.12733333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Áâ~k“‚(:à ¦„óÒ¸¾„¥) proc.memory.percent‚17.77341365814209 disk.rbd0.out‚0 network.recv‚946132( disk./.usagePercent‚76.3318823246163& disk.nvme1n1.out‚16591.36181640625 disk.dm-0.in‚ 1982.2109375 disk.dm-1.out‚ 1999.58984375" disk.nvme0c0n1.in‚ 1852.4453125 disk.nvme0n1.in‚ 1852.4453125% disk./.usageGB‚1171.8852233886719# memory_percent‚99.6058464050293# proc.memory.rssMB‚ 2911.99609375 cpu‚25.903828947593 network.sent‚ 7.706418e+06# disk.dm-0.out‚52830.40576171875 disk.rbd0.in‚0& disk.nvme0c0n1.out‚38427.732421875$ disk.nvme0n1.out‚38427.732421875% proc.memory.availableMB‚ 64.578125 proc.cpu.threads‚17 disk.nvme1n1.in‚ 129.765625 disk.md0.in‚0 disk.md0.out‚ 87.79296875 disk.dm-1.in‚0 disk.nvme1c1n1.in‚ 129.765625( disk.nvme1c1n1.out‚16591.36181640625Q{Kž”‚(:“ ¦„óÒ¿‹´¥ gpu.0.gpu‚92.0 gpu.0.memory‚17, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚50.0 gpu.0.powerWatts‚219.222' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚73.07400000000001 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0柲.2•- _wandbruntime‚3597  _runtime‚3597;ôJCÉ–Ã trainer/global_step‚5199 % _timestamp‚1.7844639231378608e+09 _runtime‚3597.526168185 _step‚424 ' lr-AdamW/pg1‚0.00011964147021191449 ' lr-AdamW/pg2‚0.00011964147021191449ëxáÎ—È _runtime‚3597.526168185 _step‚424 ' lr-AdamW/pg1‚0.00011964147021191449 ' lr-AdamW/pg2‚0.00011964147021191449 trainer/global_step‚5199 % _timestamp‚1.7844639231378608e+09¨dkþP2˜- _wandbruntime‚3597  _runtime‚3597°àRN'™ÜO \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚402755281530368 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚398010774132976 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚93380189 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 _runtime‚3597.527326386 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚407033595177984 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚402755298569728 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚11744977 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 219359232 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4278207948288 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚306006038728192 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚41136373 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚41136371 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚93390334 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚41136373 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚301601920260608 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4404337826816 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚402755281530368 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 125249536 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚402755298569728 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4266539260189 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚407033489478656 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚407033595177984 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚402277209567379 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚52243818 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚39336538 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚153 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚52253961 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 trainer/global_step‚5199 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4278296608256 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚41136371 % _timestamp‚1.7844639231389017e+09 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4278207948288 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚27591561 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚402277313393165 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚11744824 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚39336381 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚398010757093616 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 _step‚425 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚306006258087424 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4278296608256 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚301601826150912 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚52253961 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚27591557 NFDeviceStatsMšz”onitor.on_train_batch_start/inactive_split.small_pool.peak‚486 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚407033489478656ÿ $Êç'šáO OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚306006258087424 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4278296608256 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚301601826150912 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚52253961 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚27591557 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚407033489478656 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚402755281530368 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚398010774132976 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚93380189 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 _runtime‚3597.527326386 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚407033595177984 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚402755298569728 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚11744977 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 219359232 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4278207948288 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚306006038728192 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚41136373 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚41136371 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚93390334 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚41136373 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚301601920260608 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4404337826816 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚402755281530368 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 125249536 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚402755298569728 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4266539260189 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚407033489478656 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚407033595177984 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚402277209567379 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚52243818 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚39336538 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚153 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚52253961 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 trainer/global_step‚5199 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4278296608256 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚41136371 % _timestamp‚1.7844639231389017e+09 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4278207948288 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚27591561 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚402277313393165 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚11744824 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚39336381 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚398010757093616 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 _step‚425 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960©g2›- _wandbruntime‚3597  _runtime‚3597Go_ãì&œæM UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚93407797 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4279082331136 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚398085047619008 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4405149257216 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚402830448236032 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚93397638 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚93407797 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 % _timestamp‚1.7844639235546212e+09 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 135728128 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚301658057988608 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4267322754901 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚39343542 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚27596617 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚301657963878912 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚52263795 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚41144000 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚52263795 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚161 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 229837824 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚306062977408000 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚402352387413269 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4405013529088 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 _runtime‚3597.943107525àóÎìñ&ëM ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚402830448236032 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 BDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚93397638 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚93407797 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 % _timestamp‚1.7844639235546212e+09 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 135728128 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚301658057988608 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4267322754901 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚39343542 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚27596617 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚301657963878912 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚52263795 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚41144000 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚52263795 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚161 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 229837824 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚306062977408000 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚402352387413269 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4405013529088 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 _runtime‚3597.943107525 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚93407797 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4279082331136 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚398085047619008 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4405149257216ªŒ…ÌÅ2ž- _wandbruntime‚3597  _runtime‚3597-æ™ÀŸ€ 1loss/SRqqt2/detection_loss‚0.6459379947861331 .loss/FBt1/detection_loss‚0.622834427988993 trainer/global_step‚5199 2loss/SRqqt1/assignment_loss‚0.6816924229712311 /loss/FRt2/detection_loss‚1.3785810203072835 _step‚427 /loss/FRt2/assignment_loss‚1.070697251375884 2loss/FBt1/assignment_loss‚0.037671603639533986 2loss/FBt2/assignment_loss‚0.010792327630466995 /loss/FRt1/detection_loss‚0.5139272296623777 1loss/SRqqt1/detection_loss‚1.1610955754490648 /loss/FBt2/detection_loss‚0.7296935549626747 $ _timestamp‚1.784463923555614e+09 2loss/SRqqt2/assignment_loss‚0.6014590045565834 %loss/total_loss‚9.859985097881598 epoch‚12 _runtime‚3597.943696876 /loss/FRt1/assignment_loss‚2.405602684551371Åa/  … 2loss/SRqqt1/assignment_loss‚0.6816924229712311 /loss/FRt2/detection_loss‚1.3785810203072835 _step‚427 /loss/FRt2/assignment_loss‚1.070697251375884 2loss/FBt1/assignment_loss‚0.037671603639533986 2loss/FBt2/assignment_loss‚0.010792327630466995 /loss/FRt1/detection_loss‚0.5139272296623777 1loss/SRqqt1/detection_loss‚1.1610955754490648 /loss/FBt2/detection_loss‚0.7296935549626747 $ _timestamp‚1.784463923555614e+09 2loss/SRqqt2/assignment_loss‚0.6014590045565834 %loss/total_loss‚9.859985097881598 epoch‚12 _runtime‚3597.943696876 /loss/FRt1/assignment_loss‚2.405602684551371 1loss/SRqqt2/detection_loss‚0.6459379947861331 .loss/FBt1/detection_loss‚0.622834427988993 trainer/global_step‚5199«Ëd<Âô¡‚(:ä µ„óÒýÛ¤ network.sent‚ 7.712688e+06 disk.dm-1.out‚ 1999.63671875 disk.rbd0.in‚0 disk.rbd0.out‚0' disk.nvme1c1n1.out‚16593.6728515625 disk.md0.in‚0) proc.memory.percent‚17.77341365814209 proc.cpu.threads‚17 network.recv‚955950" disk.nvme0c0n1.in‚ 1852.4453125( disk.nvme0c0n1.out‚38440.27490234375 disk.nvme0n1.in‚ 1852.4453125& disk.nvme0n1.out‚38440.27490234375 disk.nvme1c1n1.in‚ 129.765625 disk.md0.out‚ 88.1171875# memory_percent‚99.6098518371582) disk./.usagePercent‚76.33191462623785% disk./.usageGB‚1171.8857192993164# disk.dm-0.out‚52844.51220703125 disk.dm-1.in‚0 disk.nvme1n1.in‚ 129.765625% disk.nvme1n1.out‚16593.6728515625% proc.meHº›]{mory.availableMB‚ 63.921875# proc.memory.rssMB‚ 2911.99609375 disk.dm-0.in‚ 1982.2109375 cpu‚25.90478937949365 D‚ž¢‚(:“ µ„óÒÿÁ£¥ gpu.0.gpu‚96.0 gpu.0.memory‚15, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚48.0 gpu.0.powerWatts‚114.145' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚38.04833333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0O»dr2£- _wandbruntime‚3611  _runtime‚3611­¨>”×=¤Ñ{ 9"Purity/2FRt*SRqqt1FBt/event_purity‚0.1133333295583725 *jet/accuracy_2_of_3‚0.5034061670303345 ( Purity/2FRt1SRqqt2FBt/FRt_purity‚0.5 7 Purity/0FRt*SRqqt1FBt/FBt_purity‚0.9960840940475464 9"Purity/0FRt1SRqqt0FBt/event_purity‚0.6370369791984558 & Purity/2FRt1SRqqt*FBt/FBt_purity‚1 9"Purity/*FRt1SRqqt*FBt/SRqqt_purity‚0.6815333366394043 8 Purity/*FRt2SRqqt*FBt/FRt_purity‚0.06456177681684494 9"Purity/0FRt1SRqqt*FBt/event_purity‚0.6280348300933838 9"Purity/0FRt2SRqqt*FBt/event_purity‚0.3636547029018402 9"Purity/1FRt*SRqqt1FBt/SRqqt_purity‚0.6451334357261658 @&Purity/1FRt1SRqqt1FBt/event_proportion‚0.0064639137126505375 <&Purity/*FRt*SRqqt0FBt/event_proportion‚0.881045401096344 =&Purity/*FRt0SRqqt0FBt/event_proportion‚0.4607747495174408 ,&Purity/*FRt2SRqqt2FBt/event_proportion‚0 ("Purity/1FRt2SRqqt1FBt/event_purity‚0 particle/accuracy_5_of_5‚0 7 Purity/*FRt*SRqqt2FBt/FBt_purity‚0.9805922508239746 9"Purity/*FRt1SRqqt0FBt/event_purity‚0.3712069094181061 8 Purity/1FRt*SRqqt*FBt/FRt_purity‚0.14053860306739807 ;"Purity/2FRt*SRqqt*FBt/event_purity‚0.015832802280783653 ;"Purity/2FRt*SRqqt0FBt/event_purity‚0.009133609011769295 ?&Purity/2FRt2SRqqt0FBt/event_proportion‚0.006417410913854837 /particle/accuracy_2_of_2‚0.9998962879180908 particle/accuracy_2_of_3‚1 8 Purity/*FRt1SRqqt0FBt/FRt_purity‚0.10723529756069183 ("Purity/0FRt0SRqqt0FBt/event_purity‚1 9"Purity/1FRt1SRqqt0FBt/SRqqt_purity‚0.7533016800880432 9"Purity/0FRt*SRqqt*FBt/SRqqt_purity‚0.6253674030303955 9 Purity/1FRt2SRqqt*FBt/FRt_purity‚0.058578260242938995 A&Purity/2FRt0SRqqt2FBt/event_proportion‚0.00018601190822664648 =&Purity/0FRt0SRqqt1FBt/event_proportion‚0.0313895083963871 /particle/accuracy_3_of_4‚0.9912415742874146 @&Purity/*FRt1SRqqt2FBt/event_proportion‚0.0006975446594879031 6 Purity/2FRt*SRqqt0FBt/FRt_purity‚0.125456303358078 8 Purity/2FRt0SRqqt0FBt/FRt_purity‚0.12825068831443787 'particle/f_score‚0.6447389125823975 @&Purity/0FRt2SRqqt1FBt/event_proportion‚0.0007207961170934141 9"Purity/*FRt*SRqqt*FBt/SRqqt_purity‚0.6776959300041199 9"Purity/*FRt*SRqqt0FBt/event_purity‚0.2342199832201004 9"Purity/*FRt*SRqqt1FBt/SRqqt_purity‚0.5697638988494873 :"Purity/*FRt0SRqqt*FBt/event_purity‚0.21021613478660583 7 Purity/1FRt1SRqqt2FBt/FRt_purity‚0.8888888955116272 jet/accuracy_2_of_5‚1 particle/accuracy_2_of_5‚1 9"Purity/1FRt*SRqqt2FBt/SRqqt_purity‚0.2222222238779068 7 Purity/1FRt1SRqqt*FBt/FBt_purity‚0.9872214794158936 9"Purity/2FRt2SRqqt0FBt/SRqqt_purity‚0.7794283032417297 7 Purity/*FRt0SRqqt*FBt/FBt_purity‚0.9947851300239563 7 Purity/*FRt0SRqqt2FBt/FRt_purity‚0.9088813066482544 8 Purity/1FRt0SRqqt0FBt/FRt_purity‚0.13868939876556396 particle/accuracy_3_of_5‚1 =&Purity/*FRt1SRqqt1FBt/event_proportion‚0.0188569575548172 :"Purity/1FRt*SRqqt*FBt/event_purity‚0.11692658811807632 ("Purity/1FRt2SRqqt*FBt/event_purity‚0 8 Purity/2FRt1SRqqt0FBt/FRt_purity‚0.15023502707481384 7 Purity/0FRt1SRqqt2FBt/FBt_purity‚0.9230769276618958 & Purity/1FRt*SRqqt1FBt/FBt_purity‚1 9"Purity/1FRt0SRqqt2FBt/event_purity‚0.8047052025794983 jet/accuracy_1_of_5‚1 9"Purity/*FRt*SRqqt2FBt/event_purity‚0.8628449440002441 ("Purity/*FRt1SRqqt2FBt/event_purity‚0 ?&Purity/0FRt1SRqqt1FBt/event_proportion‚0.011555989272892475 9"Purity/1FRt1SRqqt*FBt/SRqqt_purity‚0.7485314011573792 =&Purity/1FRt1SRqqt0FBt/event_proportion‚0.1467633992433548 7 Purity/*FRt2SRqqt1FBt/FBt_purity‚0.9814814329147339 9"Purity/0FRt*SRqqt1FBt/SRqqt_purity‚0.5254756212234497 7 Purity/0FRt*SRqqt2FBt/FBt_purity‚0.9808861613273621 /particle/accuracy_3_of_3‚0.9994933605194092 7 Purity/*FRt1SRqqt*FBt/FBt_purity‚0.9872891306877136 9"Purity/1FRt1SRqqt2FBt/SRqqt_purity‚0.2222222238779068 9"Purity/1FRt2SRqqt0FBt/SRqqt_purity‚0.7074362635612488 A&Purity/1FRt2SRqqt1FBt/event_proportion‚0.00027901786961592734 7 Purity/*FRt1SRqqt2FBt/FRt_purity‚0.8500000238418579 =&Purity/0FRt*SRqqt1FBt/event_proportion‚0.0436662957072258 8"Purity/1FRt0SRqqt*FBt/event_purity‚0.151279479265213 8 Purity/2FRt2SRqqt0FBt/FRt_purity‚0.07162931561470032 6 Purity/*FRt*SRqqt1FBt/FBt_purity‚0.997256338596344 =&Purity/*FRt1SRqqt0FBt/event_proportion‚0.3670479953289032 8 Purity/1FRt2SRqqt1FBt/FRt_purity‚0.10000000149011612 ,&Purity/1FRt2SRqqt2FBt/event_proportion‚0 9"Purity/*FRt0SRqqt1FBt/event_purity‚0.8797572255134583 =&Purity/*FRt0SRqqt*FBt/event_proportion‚0.5129278302192688 9"Purity/0FRt*SRqqt0FBt/SRqqt_purity‚0.6313695907592773 particle/accuracy_1_of_5‚1 =&Purity/0FRt*SRqqt0FBt/event_proportion‚0.2322126179933548 7 Purity/1FRt*SRqqt1FBt/FRt_purity‚0.5362011194229126 <&Purity/2FRt*SRqqt*FBt/event_proportion‚0.040062315762043 >&Purity/2FRt0SRqqt0FBt/event_proportion‚0.01990327425301075 @&Purity/2FRt0SRqqt1FBt/event_proportion‚0.0007672990905120969 +particle/sensitivity‚0.6383582353591919 ) Purity/2FRt*SRqqt2FBt/FRt_purity‚0.75 7 Purity/2FRt0SRqqt2FBt/FRt_purity‚0.7857142686843872 8"Purity/1FRt*SRqqt0FBt/SRqqt_purity‚0.742411732673645 & Purity/2FRt*SRqqt*FBt/FBt_purity‚1 9"Purity/2FRt1SRqqt0FBt/SRqqt_purity‚0.7526633739471436 & Purity/2FRt0SRqqt*FBt/FBt_purity‚1 jet/accuracy_4_of_5‚0 $ _timestamp‚1.784463937383301e+09 8 Purity/*FRt2SRqqt1FBt/FRt_purity‚0.10000000149011612 9"Purity/0FRt0SRqqt*FBt/event_purity‚0.9920928478240967 :"Purity/0FRt1SRqqt1FBt/event_purity‚0.49282678961753845 , Purity/0FRt2SRqqt*FBt/FBt_purity‚0.96875 @&Purity/2FRt*SRqqt2FBt/event_proportion‚0.0002092633949359879 ("Purity/2FRt1SRqqt1FBt/event_purity‚0 ("Purity/2FRt1SRqqt2FBt/event_purity‚0 ("Purity/*FRt2SRqqt1FBt/event_purity‚0 *jet/accuracy_1_of_2‚0.7843907475471497 7 Purity/0FRt0SRqqt*FBt/FBt_purity‚0.9935057759284973 7 Purity/1FRt0SRqqt*FBt/FBt_purity‚0.9988385438919067 9"Purity/1FRt0SRqqt1FBt/event_purity‚0.5533285140991211 @&Purity/1FRt0SRqqt2FBt/event_proportion‚0.0024879092816263437 epoch‚12 7 Purity/*FRt0SRqqt0FBt/FRt_purity‚0.1377522349357605 :"Purity/0FRt2SRqqt0FBt/event_purity‚0.37423473596572876 =&Purity/1FRt0SRqqt0FBt/event_proportion‚0.4408714771270752 5 Purity/1FRt0SRqqt2FBt/FRt_purity‚0.91700679063797 :"Purity/1FRt1SRqqt1FBt/event_purity‚0.25825855135917664 ("Purity/2FRt1SRqqt*FBt/event_purity‚0 =&Purity/*FRt*SRqqt1FBt/event_proportion‚0.0617094486951828 9"Purity/*FRt1SRqqt1FBt/SRqqt_purity‚0.5898623466491699 :"Purity/1FRt0SRqqt0FBt/event_purity‚0.13868939876556396 7 Purity/1FRt1SRqqt0FBt/FRt_purity‚0.1000240221619606 *"Purity/2FRt*SRqqt2FBt/event_purity‚0.5 jet/accuracy_5_of_5‚0 :"Purity/*FRt2SRqqt*FBt/event_purity‚0.16245011985301971 7 Purity/*FRt*SRqqt1FBt/FRt_purity‚0.5263688564300537 :"Purity/*FRt2SRqqt0FBt/event_purity‚0.16550947725772858 & Purity/1FRt2SRqqt1FBt/FBt_purity‚1 6validation_average_jet_accuracy‚0.3432621657848358 9"Purity/0FRt*SRqqt2FBt/SRqqt_purity‚0.1538461595773697 7 Purity/0FRt0SRqqt2FBt/FBt_purity‚0.9852776527404785 7 Purity/1FRt0SRqqt2FBt/FBt_purity‚0.9970238208770752 ;"Purity/1FRt1SRqqt0FBt/event_purity‚0.023477116599678993 9 Purity/1FRt2SRqqt0FBt/FRt_purity‚0.058279018849134445 8"Purity/*FRt1SRqqt0FBt/SRqqt_purity‚0.687244713306427 :"Purity/*FRt1SRqqt1FBt/event_purity‚0.39045006036758423 >&Purity/1FRt*SRqqt1FBt/event_proportion‚0.01643880270421505 & Purity/1FRt1SRqqt1FBt/FBt_purity‚1 9"Purity/2FRt*SRqqt0FBt/SRqqt_purity‚0.7639409899711609 9"Purity/2FRt2SRqqt*FBt/SRqqt_purity‚0.7794283032417297 *jet/accuracy_3_of_4‚0.0797596350312233 (particle/accuracy‚0.8489390015602112 7 Purity/*FRt0SRqqt1FBt/FBt_purity‚0.9981421232223511 9"Purity/0FRt*SRqqt2FBt/event_purity‚0.9137429594993591 9"Purity/1FRt1SRqqt1FBt/SRqqt_purity‚0.6664323210716248 8 Purity/2FRt*SRqqt*FBt/FRt_purity‚0.14328236877918243 8 Purity/2FRt0SRqqt*FBt/FRt_purity‚0.15008623898029327 +particle/specificity‚0.9065265655517578 )loss/val_total_loss‚9.723707083103616 =&Purity/*FRt*SRqqt*FBt/event_proportion‚0.9537528157234192 :"Purity/2FRt0SRqqt*FBt/event_purity‚0.03069187141954899 +jet/accuracy_3_of_3‚0.06736699491739273 ("Purity/2FRt*SRqqt2FBt/SRqqt_purity‚0 8"Purity/*FRt2SRqqt1FBt/SRqqt_purity‚0.381944477558136 9"Purity/0FRt1SRqqt0FBt/SRqqt_purity‚0.6370369791984558 +jet/accuracy_1_of_1‚0.29519447684288025 8"Purity/*FRt0SRqqt2FBt/event_purity‚0.923111081123352 :"Purity/*FRt1SRqqt*FBt/event_purity‚0.37145963311195374 ?&Purity/0FRt0SRqqt2FBt/event_proportion‚0.007626487873494625 9"Purity/0FRt1SRqqt*FBt/SRqqt_purity‚0.6308816075325012 9"Purity/0FRt1SRqqt1FBt/SRqqt_purity‚0.5403640270233154 9"Purity/0FRt2SRqqt*FBt/SRqqt_purity‚0.6030003428459167 >&Purity/0FRt2SRqqt0FBt/event_proportion‚0.02385602705180645 =&Purity/1FRt1SRqqt*FBt/event_proportion‚0.1534598171710968 8 Purity/2FRt1SRqqt1FBt/FRt_purity‚0.42824071645736694 @&Purity/2FRt1SRqqt1FBt/event_proportion‚0.0008370535797439516 +"Purity/2FRt1SRqqt1FBt/SRqqt_purity‚0.75 & Purity/2FRt*SRqqt2FBt/FBt_purity‚1 & Purity/2FRt0SRqqt2FBt/FBt_purity‚1 8 Purity/1FRt0SRqqt*FBt/FRt_purity‚0.15281149744987488 9"Purity/2FRt*SRqqt*FBt/SRqqt_purity‚0.7627214789390564 ?&Purity/2FRt1SRqqt0FBt/event_proportion‚0.011928013525903225 ("Purity/2FRt1SRqqt2FBt/SRqqt_purity‚0 7 Purity/*FRt1SRqqt2FBt/FBt_purity‚0.8657407164573669 8 Purity/1FRt1SRqqt1FBt/FRt_purity‚0.44878819584846497 7 Purity/*FRt*SRqqt0FBt/FRt_purity‚0.1263158619403839 9"Purity/0FRt*SRqqt*FBt/event_purity‚0.6546740531921387 ("Purity/1FRt1SRqqt2FBt/event_purity‚0 =&Purity/2FRt*SRqqt0FBt/event_proportion‚0.0382486991584301 ,&Purity/2FRt2SRqqt1FBt/event_proportion‚0 ,&Purity/2FRt2SRqqt2FBt/event_proportion‚0 ,jet/accuracy_4_of_4‚0.013899499550461769 particle/accuracy_1_of_1‚1 ?&Purity/2FRt2SRqqt*FBt/event_proportion‚0.006417410913854837 ("Purity/2FRt2SRqqt0FBt/event_purity‚0 7"Purity/*FRt*SRqqt0FBt/SRqqt_purity‚0.68320232629776 7 Purity/*FRt0SRqqt1FBt/FRt_purity‚0.5873658061027527 8&Purity/*FRt2SRqqt0FBt/event_proportion‚ 0.05322265625 @&Purity/2FRt*SRqqt1FBt/event_proportion‚0.0016043527284637094 +"Purity/2FRt0SRqqt1FBt/event_purity‚0.25 8 Purity/*FRt2SRqqt0FBt/FRt_purity‚0.06439140439033508 9"Purity/0FRt*SRqqt1FBt/event_purity‚0.8446938395500183 6 Purity/0FRt1SRqqt*FBt/FBt_purity‚0.987829327583313 >&Purity/0FRt2SRqqt*FBt/event_proportion‚0.02457682229578495 @&Purity/1FRt1SRqqt2FBt/event_proportion‚0.0002325148816453293 8&Purity/1FRt2SRqqt0FBt/event_proportion‚ 0.02294921875 *jet/accuracy_2_of_2‚0.1298614889383316 9"Purity/*FRt*SRqqt*FBt/event_purity‚0.2727118730545044 @&Purity/*FRt2SRqqt1FBt/event_proportion‚0.0009998140158131719 8 Purity/1FRt*SRqqt0FBt/FRt_purity‚0.12640918791294098 =&Purity/2FRt0SRqqt*FBt/event_proportion‚0.0208565853536129 =&Purity/2FRt1SRqqt*FBt/event_proportion‚0.0127883180975914 & Purity/2FRt1SRqqt1FBt/FBt_purity‚1 7 Purity/0FRt*SRqqt*FBt/FBt_purity‚0.9917944669723511 =&Purity/0FRt0SRqqt0FBt/event_proportion‚0.0462472103536129 :"Purity/*FRt*SRqqt2FBt/SRqqt_purity‚0.18518517911434174 8 Purity/*FRt1SRqqt*FBt/FRt_purity‚0.12415961176156998 :"Purity/*FRt1SRqqt2FBt/SRqqt_purity‚0.18518517911434174 ,&Purity/0FRt2SRqqt2FBt/event_proportion‚0 :"Purity/*FRt0SRqqt0FBt/event_purity‚0.13342754542827606 9"Purity/0FRt*SRqqt0FBt/event_purity‚0.6095486879348755 =&Purity/0FRt0SRqqt*FBt/event_proportion‚0.0390159972012043 7 Purity/0FRt0SRqqt1FBt/FBt_purity‚0.9974045753479004 9"Purity/1FRt*SRqqt2FBt/event_purity‚0.7339003682136536 & Purity/1FRt0SRqqt1FBt/FBt_purity‚1 7 Purity/*FRt1SRqqt1FBt/FRt_purity‚0.4517262876033783 =&Purity/1FRt*SRqqt*FBt/event_proportion‚0.6297432780265808 9"Purity/1FRt*SRqqt*FBt/SRqqt_purity‚0.7379143238067627 7 Purity/1FRt1SRqqt2FBt/FBt_purity‚0.7777777910232544 9"Purity/1FRt2SRqqt*FBt/SRqqt_purity‚0.7025189399719238 4 Purity/*FRt0SRqqt2FBt/FBt_purity‚0.9880411028862 9"Purity/*FRt*SRqqt1FBt/event_purity‚0.7151702046394348 @&Purity/1FRt*SRqqt2FBt/event_proportion‚0.0027204242069274187 :"Purity/1FRt1SRqqt*FBt/event_purity‚0.03346550837159157 >&Purity/1FRt2SRqqt*FBt/event_proportion‚0.02322823740541935 & Purity/1FRt2SRqqt*FBt/FBt_purity‚1 ("Purity/1FRt2SRqqt0FBt/event_purity‚0 ?&Purity/0FRt*SRqqt2FBt/event_proportion‚0.008068266324698925 ?&Purity/*FRt*SRqqt2FBt/event_proportion‚0.010997953824698925 8 Purity/*FRt0SRqqt*FBt/FRt_purity‚0.15248052775859833 =&Purity/0FRt*SRqqt*FBt/event_proportion‚0.2839471697807312 @&Purity/0FRt1SRqqt2FBt/event_proportion‚0.0004417782765813172 & Purity/2FRt*SRqqt1FBt/FBt_purity‚1 ;"Purity/2FRt0SRqqt0FBt/event_purity‚0.017657285556197166 & Purity/2FRt0SRqqt1FBt/FBt_purity‚1 =&Purity/*FRt0SRqqt2FBt/event_proportion‚0.0103004090487957 9"Purity/0FRt0SRqqt1FBt/event_purity‚0.9974045753479004 , Purity/0FRt2SRqqt1FBt/FBt_purity‚0.96875 7 Purity/1FRt1SRqqt*FBt/FRt_purity‚0.1165844053030014 particle/accuracy_2_of_4‚1 6 Purity/*FRt1SRqqt1FBt/FBt_purity‚0.996301531791687 9"Purity/*FRt2SRqqt0FBt/SRqqt_purity‚0.6696321368217468 8 Purity/2FRt1SRqqt*FBt/FRt_purity‚0.16841670870780945 *jet/accuracy_1_of_3‚0.9180680513381958 particle/accuracy_1_of_4‚1 9"Purity/2FRt0SRqqt2FBt/event_purity‚0.5714285969734192 jet/accuracy_3_of_5‚1 =&Purity/*FRt0SRqqt1FBt/event_proportion‚0.0418526791036129 <&Purity/*FRt2SRqqt*FBt/event_proportion‚0.054222472012043 7 Purity/1FRt0SRqqt1FBt/FRt_purity‚0.5969142913818359 *jet/accuracy_1_of_4‚0.9507973790168762 7 Purity/*FRt*SRqqt*FBt/FBt_purity‚0.9927864074707031 =&Purity/0FRt1SRqqt*FBt/event_proportion‚0.2203543484210968 9"Purity/0FRt2SRqqt0FBt/SRqqt_purity‚0.6087010502815247 :"Purity/1FRt2SRqqt1FBt/SRqqt_purity‚0.32499998807907104 /particle/accuracy_4_of_4‚0.9501655101776123 7 Purity/*FRt*SRqqt2FBt/FRt_purity‚0.9030990600585938 :"Purity/1FRt*SRqqt1FBt/event_purity‚0.43021923303604126 trainer/global_step‚5199 =&Purity/*FRt1SRqqt*FBt/event_proportion‚0.3866024911403656 =&Purity/0FRt1SRqqt0FBt/event_proportion‚0.2083565890789032 7 Purity/1FRt*SRqqt2FBt/FRt_purity‚0.9133219718933105 8 Purity/2FRt*SRqqt1FBt/FRt_purity‚0.48083335161209106 9"Purity/2FRt1SRqqt*FBt/SRqqt_purity‚0.7519316673278809 particle/accuracy_1_of_3‚1 . Purity/2FRt0SRqqt1FBt/FRt_purity‚ 0.5703125 ("Purity/0FRt2SRqqt1FBt/event_purity‚0 7 Purity/1FRt*SRqqt*FBt/FBt_purity‚0.9944894909858704 +"Purity/2FRt*SRqqt1FBt/SRqqt_purity‚0.75 8 Purity/2FRt2SRqqt*FBt/FRt_purity‚0.07162931561470032 9"Purity/0FRt1SRqqt2FBt/SRqqt_purity‚0.1538461595773697 _step‚428 8"Purity/*FRt2SRqqt*FBt/SRqqt_purity‚0.664395272731781 :"Purity/1FRt*SRqqt0FBt/event_purity‚0.10571011155843735 =&Purity/1FRt0SRqqt*FBt/event_proportion‚0.4530552327632904 ("Purity/2FRt1SRqqt0FBt/event_purity‚0 @&Purity/2FRt1SRqqt2FBt/event_proportion‚2.325148852833081e-05 _runtime‚3611.772141274 7 Purity/*FRt2SRqqt*FBt/FBt_purity‚0.9814814329147339 >&Purity/1FRt0SRqqt1FBt/event_proportion‚0.00969587080180645 & Purity/2FRt1SRqqt2FBt/FBt_purity‚1 particle/accuracy_4_of_5‚1 9"Purity/0FRt0SRqqt2FBt/event_purity‚0.9705552458763123 7 Purity/0FRt1SRqqt1FBt/FBt_purity‚0.9940876960754395 =&Purity/1FRt*SRqqt0FBt/event_proportion‚0.6105840802192688 ("Purity/2FRt2SRqqt*FBt/event_purity‚0 particle/accuracy_1_of_2‚1 ("Purity/0FRt1SRqqt2FBt/event_purity‚0 6 Purity/*FRt*SRqqt*FBt/FRt_purity‚0.140831857919693 7 Purity/1FRt*SRqqt2FBt/FBt_purity‚0.9795351624488831 *jet/accuracy_2_of_4‚0.7420719861984253 /"Purity/0FRt2SRqqt1FBt/SRqqt_purity‚0.421875¹²™îÜ=¥Ö{ particle/accuracy_2_of_4‚1 6 Purity/*FRt1SRqqt1FBt/FBt_purity‚0.996301531791687 9"Purity/*FRt2SRqqt0FBt/SRqqt_purity‚0.6696321368217468 8 Purity/2FRt1SRqqt*FBt/FRt_purity‚0.16841670870780945 *jet/accuracy_1_of_3‚0.9180680513381958 particle/accuracy_1_of_4‚1 9"Purity/2FRt0SRqqt2FBt/event_purity‚0.5714285969734192 jet/accuracy_3_of_5‚1 =&Purity/*FRt0SRqqt1FBt/event_proportion‚0.0418526791036129 <&Purity/*FRt2SRqqt*FBt/event_proportion‚0.054222472012043 7 Purity/1FRt0SRqqt1FBt/FRt_purity‚0.5969142913818359 *jet/accuracy_1_of_4‚0.9507973790168762 7 Purity/*FRt*SRqqt*FBt/FBt_purity‚0.9927864074707031 =&Purity/0FRt1SRqqt*FBt/event_proportion‚0.2203543484210968 9"Purity/0FRt2SRqqt0FBt/SRqqt_purity‚0.6087010502815247 :"Purity/1FRt2SRqqt1FBt/SRqqt_purity‚0.32499998807907104 /particle/accuracy_4_of_4‚0.9501655101776123 7 Purity/*FRt*SRqqt2FBt/FRt_purity‚0.9030990600585938 :"Purity/1FRt*SRqqt1FBt/event_purity‚0.43021923303604126 trainer/global_step‚5199 =&Purity/*FRt1SRqqt*FBt/event_proportion‚0.3866024911403656 =&Purity/0FRt1SRqqt0FBt/event_proportion‚0.2083565890789032 7 Purity/1FRt*SRqqt2FBt/FRt_purity‚0.9133219718933105 8 Purity/2FRt*SRqqt1FBt/FRt_purity‚0.48083335161209106 9"Purity/2FRt1SRqqt*FBt/SRqqt_purity‚0.7519316673278809 particle/accuracy_1_of_3‚1 . Purity/2FRt0SRqqt1FBt/FRt_purity‚ 0.5703125 ("Purity/0FRt2SRqqt1FBt/event_purity‚0 7 Purity/1FRt*SRqqt*FBt/FBt_purity‚0.9944894909858704 +"Purity/2FRt*SRqqt1FBt/SRqqt_purity‚0.75 8 Purity/2FRt2SRqqt*FBt/FRt_purity‚0.07162931561470032 9"Purity/0FRt1SRqqt2FBt/SRqqt_purity‚0.1538461595773697 _step‚428 8"Purity/*FRt2SRqqt*FBt/SRqqt_purity‚0.664395272731781 :"Purity/1FRt*SRqqt0FBt/event_purity‚0.10571011155843735 =&Purity/1FRt0SRqqt*FBt/event_proportion‚0.4530552327632904 ("Purity/2FRt1SRqqt0FBt/event_purity‚0 @&Purity/2FRt1SRqqt2FBt/event_proportion‚2.325148852833081e-05 _runtime‚3611.772141274 7 Purity/*FRt2SRqqt*FBt/FBt_purity‚0.9814814329147339 >&Purity/1FRt0SRqqt1FBt/event_proportion‚0.00969587080180645 & Purity/2FRt1SRqqt2FBt/FBt_purity‚1 particle/accuracy_4_of_5‚1 9"Purity/0FRt0SRqqt2FBt/event_purity‚0.9705552458763123 7 Purity/0FRt1SRqqt1FBt/FBt_purity‚0.9940876960754395 =&Purity/1FRt*SRqqt0FBt/event_proportion‚0.6105840802192688 ("Purity/2FRt2SRqqt*FBt/event_purity‚0 particle/accuracy_1_of_2‚1 ("Purity/0FRt1SRqqt2FBt/event_purity‚0 6 Purity/*FRt*SRqqt*FBt/FRt_purity‚0.140831857919693 7 Purity/1FRt*SRqqt2FBt/FBt_purity‚0.9795351624488831 *jet/accuracy_2_of_4‚0.7420719861984253 /"Purity/0FRt2SRqqt1FBt/SRqqt_purity‚0.421875 9"Purity/2FRt*SRqqt1FBt/event_purity‚0.1133333295583725 *jet/accuracy_2_of_3‚0.5034061670303345 ( Purity/2FRt1SRqqt2FBt/FRt_purity‚0.5 7 Purity/0FRt*SRqqt1FBt/FBt_purity‚0.9960840940475464 9"Purity/0FRt1SRqqt0FBt/event_purity‚0.6370369791984558 & Purity/2FRt1SRqqt*FBt/FBt_purity‚1 9"Purity/*FRt1SRqqt*FBt/SRqqt_purity‚0.6815333366394043 8 Purity/*FRt2SRqqt*FBt/FRt_purity‚0.06456177681684494 9"Purity/0FRt1SRqqt*FBt/event_purity‚0.6280348300933838 9"Purity/0FRt2SRqqt*FBt/event_purity‚0.3636547029018402 9"Purity/1FRt*SRqqt1FBt/SRqqt_purity‚0.6451334357261658 @&Purity/1FRt1SRqqt1FBt/event_proportion‚0.0064639137126505375 <&Purity/*FRt*SRqqt0FBt/event_proportion‚0.881045401096344 =&Purity/*FRt0SRqqt0FBt/event_proportion‚0.4607747495174408 ,&Purity/*FRt2SRqqt2FBt/event_proportion‚0 ("Purity/1FRt2SRqqt1FBt/event_purity‚0 particle/accuracy_5_of_5‚0 7 Purity/*FRt*SRqqt2FBt/FBt_purity‚0.9805922508239746 9"Purity/*FRt1SRqqt0FBt/event_purity‚0.3712069094181061 8 Purity/1FRt*SRqqt*FBt/FRt_purity‚0.14053860306739807 ;"Purity/2FRt*SRqqt*FBt/event_purity‚0.015832802280783653 ;"Purity/2FRt*SRqqt0FBt/event_purity‚0.009133609011769295 ?&Purity/2FRt2SRqqt0FBt/event_proportion‚0.006417410913854837 /particle/accuracy_2_of_2‚0.9998962879180908 particle/accuracy_2_of_3‚1 8 Purity/*FRt1SRqqt0FBt/FRt_purity‚0.10723529756069183 ("Purity/0FRt0SRqqt0FBt/event_purity‚1 9"Purity/1FRt1SRqqt0FBt/SRqqt_purity‚0.7533016800880432 9"Purity/0FRt*SRqqt*FBt/SRqqt_purity‚0.6253674030303955 9 Purity/1FRt2SRqqt*FBt/FRt_purity‚0.058578260242938995 A&Purity/2FRt0SRqqt2FBt/event_proportion‚0.00018601190822664648 =&Purity/0FRt0SRqqt1FBt/event_proportion‚0.0313895083963871 /particle/accuracy_3_of_4‚0.9912415742874146 @&Purity/*FRt1SRqqt2FBt/event_proportion‚0.0006975446594879031 6 Purity/2FRt*SRqqt0FBt/FRt_purity‚0.125456303358078 8 Purity/2FRt0SRqqt0FBt/FRt_purity‚0.12825068831443787 'particle/f_score‚0.6447389125823975 @&Purity/0FRt2SRqqt1FBt/event_proportion‚0.0007207961170934141 9"Purity/*FRt*SRqqt*FBt/SRqqt_purity‚0.6776959300041199 9"Purity/*FRt*SRqqt0FBt/event_purity‚0.2342199832201004 9"Purity/*FRt*SRqqt1FBt/SRqqt_purity‚0.5697638988494873 :"Purity/*FRt0SRqqt*FBt/event_purity‚0.21021613478660583 7 Purity/1FRt1SRqqt2FBt/FRt_purity‚0.8888888955116272 jet/accuracy_2_of_5‚1 particle/accuracy_2_of_5‚1 9"Purity/1FRt*SRqqt2FBt/SRqqt_purity‚0.2222222238779068 7 Purity/1FRt1SRqqt*FBt/FBt_purity‚0.9872214794158936 9"Purity/2FRt2SRqqt0FBt/SRqqt_purity‚0.7794283032417297 7 Purity/*FRt0SRqqt*FBt/FBt_purity‚0.9947851300239563 7 Purity/*FRt0SRqqt2FBt/FRt_purity‚0.9088813066482544 8 Purity/1FRt0SRqqt0FBt/FRt_purity‚0.13868939876556396 particle/accuracy_3_of_5‚1 =&Purity/*FRt1SRqqt1FBt/event_proportion‚0.0188569575548172 :"Purity/1FRt*SRqqt*FBt/event_purity‚0.11692658811807632 ("Purity/1FRt2SRqqt*FBt/event_purity‚0 8 Purity/2FRt1SRqqt0FBt/FRt_purity‚0.15023502707481384 7 Purity/0FRt1SRqqt2FBt/FBt_purity‚0.9230769276618958 & Purity/1FRt*SRqqt1FBt/FBt_purity‚1 9"Purity/1FRt0SRqqt2FBt/event_purity‚0.8047052025794983 jet/accuracy_1_of_5‚1 9"Purity/*FRt*SRqqt2FBt/event_purity‚0.8628449440002441 ("Purity/*FRt1SRqqt2FBt/event_purity‚0 ?&Purity/0FRt1SRqqt1FBt/event_proportion‚0.011555989272892475 9"Purity/1FRt1SRqqt*FBt/SRqqt_purity‚0.7485314011573792 =&Purity/1FRt1SRqqt0FBt/event_proportion‚0.1467633992433548 7 Purity/*FRt2SRqqt1FBt/FBt_purity‚0.9814814329147339 9"Purity/0FRt*SRqqt1FBt/SRqqt_purity‚0.5254756212234497 7 Purity/0FRt*SRqqt2FBt/FBt_purity‚0.9808861613273621 /particle/accuracy_3_of_3‚0.9994933605194092 7 Purity/*FRt1SRqqt*FBt/FBt_purity‚0.9872891306877136 9"Purity/1FRt1SRqqt2FBt/SRqqt_purity‚0.2222222238779068 9"Purity/1FRt2SRqqt0FBt/SRqqt_purity‚0.7074362635612488 A&Purity/1FRt2SRqqt1FBt/event_proportion‚0.00027901786961592734 7 Purity/*FRt1SRqqt2FBt/FRt_purity‚0.8500000238418579 =&Purity/0FRt*SRqqt1FBt/event_proportion‚0.0436662957072258 8"Purity/1FRt0SRqqt*FBt/event_purity‚0.151279479265213 8 Purity/2FRt2SRqqt0FBt/FRt_purity‚0.07162931561470032 6 Purity/*FRt*SRqqt1FBt/FBt_purity‚0.997256338596344 =&Purity/*FRt1SRqqt0FBt/event_proportion‚0.3670479953289032 8 Purity/1FRt2SRqqt1FBt/FRt_purity‚0.10000000149011612 ,&Purity/1FRt2SRqqt2FBt/event_proportion‚0 9"Purity/*FRt0SRqqt1FBt/event_purity‚0.8797572255134583 =&Purity/*FRt0SRqqt*FBt/event_proportion‚0.5129278302192688 9"Purity/0FRt*SRqqt0FBt/SRqqt_purity‚0.6313695907592773 particle/accuracy_1_of_5‚1 =&Purity/0FRt*SRqqt0FBt/event_proportion‚0.2322126179933548 7 Purity/1FRt*SRqqt1FBt/FRt_purity‚0.5362011194229126 <&Purity/2FRt*SRqqt*FBt/event_proportion‚0.040062315762043 >&Purity/2FRt0SRqqt0FBt/event_proportion‚0.01990327425301075 @&Purity/2FRt0SRqqt1FBt/event_proportion‚0.0007672990905120969 +particle/sensitivity‚0.6383582353591919 ) Purity/2FRt*SRqqt2FBt/FRt_purity‚0.75 7 Purity/2FRt0SRqqt2FBt/FRt_purity‚0.7857142686843872 8"Purity/1FRt*SRqqt0FBt/SRqqt_purity‚0.742411732673645 & Purity/2FRt*SRqqt*FBt/FBt_purity‚1 9"Purity/2FRt1SRqqt0FBt/SRqqt_purity‚0.7526633739471436 & Purity/2FRt0SRqqt*FBt/FBt_purity‚1 jet/accuracy_4_of_5‚0 $ _timestamp‚1.784463937383301e+09 8 Purity/*FRt2SRqqt1FBt/FRt_purity‚0.10000000149011612 9"Purity/0FRt0SRqqt*FBt/event_purity‚0.9920928478240967 :"Purity/0FRt1SRqqt1FBt/event_purity‚0.49282678961753845 , Purity/0FRt2SRqqt*FBt/FBt_purity‚0.96875 @&Purity/2FRt*SRqqt2FBt/event_proportion‚0.0002092633949359879 ("Purity/2FRt1SRqqt1FBt/event_purity‚0 ("Purity/2FRt1SRqqt2FBt/event_purity‚0 ("Purity/*FRt2SRqqt1FBt/event_purity‚0 *jet/accuracy_1_of_2‚0.7843907475471497 7 Purity/0FRt0SRqqt*FBt/FBt_purity‚0.9935057759284973 7 Purity/1FRt0SRqqt*FBt/FBt_purity‚0.9988385438919067 9"Purity/1FRt0SRqqt1FBt/event_purity‚0.5533285140991211 @&Purity/1FRt0SRqqt2FBt/event_proportion‚0.0024879092816263437 epoch‚12 7 Purity/*FRt0SRqqt0FBt/FRt_purity‚0.1377522349357605 :"Purity/0FRt2SRqqt0FBt/event_purity‚0.37423473596572876 =&Purity/1FRt0SRqqt0FBt/event_proportion‚0.4408714771270752 5 Purity/1FRt0SRqqt2FBt/FRt_purity‚0.91700679063797 :"Purity/1FRt1SRqqt1FBt/event_purity‚0.25825855135917664 ("Purity/2FRt1SRqqt*FBt/event_purity‚0 =&Purity/*FRt*SRqqt1FBt/event_proportion‚0.0617094486951828 9"Purity/*FRt1SRqqt1FBt/SRqqt_purity‚0.5898623466491699 :"Purity/1FRt0SRqqt0FBt/event_purity‚0.13868939876556396 7 Purity/1FRt1SRqqt0FBt/FRt_purity‚0.1000240221619606 *"Purity/2FRt*SRqqt2FBt/event_purity‚0.5 jet/accuracy_5_of_5‚0 :"Purity/*FRt2SRqqt*FBt/event_purity‚0.16245011985301971 7 Purity/*FRt*SRqqt1FBt/FRt_purity‚0.5263688564300537 :"Purity/*FRt2SRqqt0FBt/event_purity‚0.16550947725772858 & Purity/1FRt2SRqqt1FBt/FBt_purity‚1 6validation_average_jet_accuracy‚0.3432621657848358 9"Purity/0FRt*SRqqt2FBt/SRqqt_purity‚0.1538461595773697 7 Purity/0FRt0SRqqt2FBt/FBt_purity‚0.9852776527404785 7 Purity/1FRt0SRqqt2FBt/FBt_purity‚0.9970238208770752 ;"Purity/1FRt1SRqqt0FBt/event_purity‚0.023477116599678993 9 Purity/1FRt2SRqqt0FBt/FRt_purity‚0.058279018849134445 8"Purity/*FRt1SRqqt0FBt/SRqqt_purity‚0.687244713306427 :"Purity/*FRt1SRqqt1FBt/event_purity‚0.39045006036758423 >&Purity/1FRt*SRqqt1FBt/event_proportion‚0.01643880270421505 & Purity/1FRt1SRqqt1FBt/FBt_purity‚1 9"Purity/2FRt*SRqqt0FBt/SRqqt_purity‚0.7639409899711609 9"Purity/2FRt2SRqqt*FBt/SRqqt_purity‚0.7794283032417297 *jet/accuracy_3_of_4‚0.0797596350312233 (particle/accuracy‚0.8489390015602112 7 Purity/*FRt0SRqqt1FBt/FBt_purity‚0.9981421232223511 9"Purity/0FRt*SRqqt2FBt/event_purity‚0.9137429594993591 9"Purity/1FRt1SRqqt1FBt/SRqqt_purity‚0.6664323210716248 8 Purity/2FRt*SRqqt*FBt/FRt_purity‚0.14328236877918243 8 Purity/2FRt0SRqqt*FBt/FRt_purity‚0.15008623898029327 +particle/specificity‚0.9065265655517578 )loss/val_total_loss‚9.723707083103616 =&Purity/*FRt*SRqqt*FBt/event_proportion‚0.9537528157234192 :"Purity/2FRt0SRqqt*FBt/event_purity‚0.03069187141954899 +jet/accuracy_3_of_3‚0.06736699491739273 ("Purity/2FRt*SRqqt2FBt/SRqqt_purity‚0 8"Purity/*FRt2SRqqt1FBt/SRqqt_purity‚0.381944477558136 9"Purity/0FRt1SRqqt0FBt/SRqqt_purity‚0.6370369791984558 +jet/accuracy_1_of_1‚0.29519447684288025 8"Purity/*FRt0SRqqt2FBt/event_purity‚0.923111081123352 :"Purity/*FRt1SRqqt*FBt/event_purity‚0.37145963311195374 ?&Purity/0FRt0SRqqt2FBt/event_proportion‚0.007626487873494625 9"Purity/0FRt1SRqqt*FBt/SRqqt_purity‚0.6308816075325012 9"Purity/0FRt1SRqqt1FBt/SRqqt_purity‚0.5403640270233154 9"Purity/0FRt2SRqqt*FBt/SRqqt_purity‚0.6030003428459167 >&Purity/0FRt2SRqqt0FBt/event_proportion‚0.02385602705180645 =&Purity/1FRt1SRqqt*FBt/event_proportion‚0.1534598171710968 8 Purity/2FRt1SRqqt1FBt/FRt_purity‚0.42824071645736694 @&Purity/2FRt1SRqqt1FBt/event_proportion‚0.0008370535797439516 +"Purity/2FRt1SRqqt1FBt/SRqqt_purity‚0.75 & Purity/2FRt*SRqqt2FBt/FBt_purity‚1 & Purity/2FRt0SRqqt2FBt/FBt_purity‚1 8 Purity/1FRt0SRqqt*FBt/FRt_purity‚0.15281149744987488 9"Purity/2FRt*SRqqt*FBt/SRqqt_purity‚0.7627214789390564 ?&Purity/2FRt1SRqqt0FBt/event_proportion‚0.011928013525903225 ("Purity/2FRt1SRqqt2FBt/SRqqt_purity‚0 7 Purity/*FRt1SRqqt2FBt/FBt_purity‚0.8657407164573669 8 Purity/1FRt1SRqqt1FBt/FRt_purity‚0.44878819584846497 7 Purity/*FRt*SRqqt0FBt/FRt_purity‚0.1263158619403839 9"Purity/0FRt*SRqqt*FBt/event_purity‚0.6546740531921387 ("Purity/1FRt1SRqqt2FBt/event_purity‚0 =&Purity/2FRt*SRqqt0FBt/event_proportion‚0.0382486991584301 ,&Purity/2FRt2SRqqt1FBt/event_proportion‚0 ,&Purity/2FRt2SRqqt2FBt/event_proportion‚0 ,jet/accuracy_4_of_4‚0.013899499550461769 particle/accuracy_1_of_1‚1 ?&Purity/2FRt2SRqqt*FBt/event_proportion‚0.006417410913854837 ("Purity/2FRt2SRqqt0FBt/event_purity‚0 7"Purity/*FRt*SRqqt0FBt/SRqqt_purity‚0.68320232629776 7 Purity/*FRt0SRqqt1FBt/FRt_purity‚0.5873658061027527 8&Purity/*FRt2SRqqt0FBt/event_proportion‚ 0.05322265625 @&Purity/2FRt*SRqqt1FBt/event_proportion‚0.0016043527284637094 +"Purity/2FRt0SRqqt1FBt/event_purity‚0.25 8 Purity/*FRt2SRqqt0FBt/FRt_purity‚0.06439140439033508 9"Purity/0FRt*SRqqt1FBt/event_purity‚0.8446938395500183 6 Purity/0FRt1SRqqt*FBt/FBt_purity‚0.987829327583313 >&Purity/0FRt2SRqqt*FBt/event_proportion‚0.02457682229578495 @&Purity/1FRt1SRqqt2FBt/event_proportion‚0.0002325148816453293 8&Purity/1FRt2SRqqt0FBt/event_proportion‚ 0.02294921875 *jet/accuracy_2_of_2‚0.1298614889383316 9"Purity/*FRt*SRqqt*FBt/event_purity‚0.2727118730545044 @&Purity/*FRt2SRqqt1FBt/event_proportion‚0.0009998140158131719 8 Purity/1FRt*SRqqt0FBt/FRt_purity‚0.12640918791294098 =&Purity/2FRt0SRqqt*FBt/event_proportion‚0.0208565853536129 =&Purity/2FRt1SRqqt*FBt/event_proportion‚0.0127883180975914 & Purity/2FRt1SRqqt1FBt/FBt_purity‚1 7 Purity/0FRt*SRqqt*FBt/FBt_purity‚0.9917944669723511 =&Purity/0FRt0SRqqt0FBt/event_proportion‚0.0462472103536129 :"Purity/*FRt*SRqqt2FBt/SRqqt_purity‚0.18518517911434174 8 Purity/*FRt1SRqqt*FBt/FRt_purity‚0.12415961176156998 :"Purity/*FRt1SRqqt2FBt/SRqqt_purity‚0.18518517911434174 ,&Purity/0FRt2SRqqt2FBt/event_proportion‚0 :"Purity/*FRt0SRqqt0FBt/event_purity‚0.13342754542827606 9"Purity/0FRt*SRqqt0FBt/event_purity‚0.6095486879348755 =&Purity/0FRt0SRqqt*FBt/event_proportion‚0.0390159972012043 7 Purity/0FRt0SRqqt1FBt/FBt_purity‚0.9974045753479004 9"Purity/1FRt*SRqqt2FBt/event_purity‚0.7339003682136536 & Purity/1FRt0SRqqt1FBt/FBt_purity‚1 7 Purity/*FRt1SRqqt1FBt/FRt_purity‚0.4517262876033783 =&Purity/1FRt*SRqqt*FBt/event_proportion‚0.6297432780265808 9"Purity/1FRt*SRqqt*FBt/SRqqt_purity‚0.7379143238067627 7 Purity/1FRt1SRqqt2FBt/FBt_purity‚0.7777777910232544 9"Purity/1FRt2SRqqt*FBt/SRqqt_purity‚0.7025189399719238 4 Purity/*FRt0SRqqt2FBt/FBt_purity‚0.9880411028862 9"Purity/*FRt*SRqqt1FBt/event_purity‚0.7151702046394348 @&Purity/1FRt*SRqqt2FBt/event_proportion‚0.0027204242069274187 :"Purity/1FRt1SRqqt*FBt/event_purity‚0.03346550837159157 >&Purity/1FRt2SRqqt*FBt/event_proportion‚0.02322823740541935 & Purity/1FRt2SRqqt*FBt/FBt_purity‚1 ("Purity/1FRt2SRqqt0FBt/event_purity‚0 ?&Purity/0FRt*SRqqt2FBt/event_proportion‚0.008068266324698925 ?&Purity/*FRt*SRqqt2FBt/event_proportion‚0.010997953824698925 8 Purity/*FRt0SRqqt*FBt/FRt_purity‚0.15248052775859833 =&Purity/0FRt*SRqqt*FBt/event_proportion‚0.2839471697807312 @&Purity/0FRt1SRqqt2FBt/event_proportion‚0.0004417782765813172 & Purity/2FRt*SRqqt1FBt/FBt_purity‚1 ;"Purity/2FRt0SRqqt0FBt/event_purity‚0.017657285556197166 & Purity/2FRt0SRqqt1FBt/FBt_purity‚1 =&Purity/*FRt0SRqqt2FBt/event_proportion‚0.0103004090487957 9"Purity/0FRt0SRqqt1FBt/event_purity‚0.9974045753479004 , Purity/0FRt2SRqqt1FBt/FBt_purity‚0.96875 7 Purity/1FRt1SRqqt*FBt/FRt_purity‚0.1165844053030014¬;0jhئ‚(:ë Ä„óÒ’òŽ¥ cpu‚25.91344203259763% disk./.usageGB‚1171.8857803344727 disk.dm-0.in‚ 1982.2109375" disk.dm-1.out‚1999.80908203125( disk.nvme0c0n1.out‚38441.38623046875 disk.nvme1c1n1.in‚ 129.765625" memory_percent‚99.619460105896 proc.cpu.threads‚17 network.sent‚ 7.788175e+06& disk.nvme1n1.out‚16594.59716796875# disk.dm-0.out‚52845.13720703125 disk.dm-1.in‚0 disk.rbd0.in‚0& disk.nvme0n1.out‚38441.Ö±úEž38623046875 disk.rbd0.out‚0 network.recv‚959679) disk./.usagePercent‚76.33191860182203 disk.nvme1n1.in‚ 129.765625 disk.md0.in‚0 disk.md0.out‚ 88.70703125' proc.memory.availableMB‚ 62.34765625# proc.memory.rssMB‚ 2913.99609375" disk.nvme0c0n1.in‚ 1852.4453125 disk.nvme0n1.in‚ 1852.4453125( disk.nvme1c1n1.out‚16594.59716796875) proc.memory.percent‚17.785620689392096 §‚(:… Ä„óÒëÁ¥ gpu.0.gpu‚43.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚49.0 gpu.0.powerWatts‚224.07' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚74.69 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0z•q¨‚(:æ Ó„óÒæЃ¥ disk.md0.out‚ 88.91796875# disk.dm-0.out‚52846.28955078125& proc.memory.availableMB‚ 62.3046875 proc.cpu.threads‚17 network.sent‚ 7.831866e+06 disk.rbd0.in‚0 disk.rbd0.out‚0% disk.nvme0n1.out‚38442.6845703125' disk.nvme1c1n1.out‚16594.9931640625 disk.nvme1n1.in‚ 129.765625 disk.md0.in‚0# proc.memory.rssMB‚ 2913.99609375 disk.dm-1.in‚0" disk.nvme0c0n1.in‚ 1852.4453125' disk.nvme0c0n1.out‚38442.6845703125 disk.nvme1c1n1.in‚ 129.765625 disk.dm-0.in‚ 1982.2109375) proc.memory.percent‚17.78562068939209 network.recv‚963836' disk./.usagePercent‚76.331918104874" disk.dm-1.out‚1999.88720703125$ memory_percent‚99.61972236633301 cpu‚25.91201569339249% disk./.usageGB‚1171.8857727050781 disk.nvme0n1.in‚ 1852.4453125% disk.nvme1n1.out‚16594.9931640625·»SÑœ©‚(:‘ Ó„óÒôø²¥ gpu.0.gpu‚43.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚49.0 gpu.0.powerWatts‚226.95' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚75.64999999999999 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0 ö&¯xª‚(:í â„óÒæ°è¤ disk.rbd0.out‚0 disk.nvme0n1.in‚ 1852.4453125 disk.nvme1n1.in‚ 129.765625# proc.memory.rssMB‚ 2913.99609375 cpu‚25.910534583696595 network.sent‚ 7.841709e+06 disk.rbd0.in‚0& disk.nvme0n1.out‚38521.04931640625 disk.dm-0.in‚ 1982.2109375# memory_percent‚99.6196985244751 disk.nvme1c1n1.in‚ 129.765625( disk.nvme1c1n1.out‚16606.07958984375& disk.nvme1n1.out‚16606.07958984375 disk.dm-1.in‚0" disk.dm-1.out‚2000.06982421875) proc.memory.percent‚17.78562068939209 proc.cpu.threads‚17 network.recv‚974079% disk./.usageGB‚1172.1751976013184" disk.nvme0c0n1.in‚ 1852.4453125( disk.nvme0c0n1.out‚38521.04931640625 disk.md0.in‚0 disk.md0.out‚ 89.22265625# disk.dm-0.out‚52934.88330078125' proc.memory.availableMB‚ 62.30859375) disk./.usagePercent‚76.35077007662065¼æ²\’«‚(:‡ â„óÒ匚¥ gpu.0.gpu‚31.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚47.0 gpu.0.powerWatts‚114.024' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚38.008 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ŠK=p2¬- _wandbruntime‚3645  _runtime‚3645žh)É­Ã trainer/global_step‚5249 % _timestamp‚1.7844639707012758e+09 _runtime‚3645.089672193 _step‚429 ' lr-AdamW/pg1‚0.00011925176669112809 ' lr-AdamW/pg2‚0.00011925176669112809Ï|ÞbÎ®È trainer/global_step‚5249 % _timestamp‚1.7844639707012758e+09 _runtime‚3645.089672193 _step‚429 ' lr-AdamW/pg1‚0.00011925176669112809 ' lr-AdamW/pg2‚0.00011925176669112809­þ<¤:2¯- _wandbruntime‚3645  _runtime‚3645\vFâ'°ÜO WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚185 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚411842239404544 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4321106263132 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚411842345103872 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚41639623 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚94458793 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4333075922944 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4332987262976 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4332987262976 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚39850893 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚27952612 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚407019960488326 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚407509252141568 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚305212171865600 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚305212077755904 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4321193049558 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚411842239404544 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚407509269180928 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚407509252141568 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 % _timestamp‚1.7844639707023895e+09 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4333075922944 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚27952616 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚11898466 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 trainer/global_step‚5249 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚41639623 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4460527576064 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚94458793 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4333075922944 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4332987262976 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4332987262976 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚39850893 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚27952612 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚407019960488326 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚407509252141568 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚305212171865600 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚305212077755904 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4321193049558 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚411842239404544 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚407509269180928 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚407509252141568 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 % _timestamp‚1.7844639707023895e+09 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4333075922944 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚27952616 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚11898466 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 trainer/global_step‚5249 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚41639623 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4460527576064 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚411842239404544 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4321106263132 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚411842345103872 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚41639623 IDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚39858187 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚407584401807872 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚27957672 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚94486401 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚309729438685184 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚407094930682532 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚94476242 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4461344249344 % _timestamp‚1.7844639711274943e+09 _runtime‚3645.516074293 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚94486401 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚407584401807872 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚11900701 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚182 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚41647250 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚411918174786560 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4321976544270 _step‚431 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4333861645824 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚41647250 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚11900519 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 215157760 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚41647252 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚41647252 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚27957668 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚402773040924800 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 trainer/global_step‚5249 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4333772978688 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚407584418847232 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚94476242 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚407095034508430 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚39858373 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚309729653842944 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4333772978688 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚30526821548]r퀉3904 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚52828992 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 121048064 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4461223201280 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚186 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚411918280493056 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚182 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚41647250 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚411918174786560 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4321976544270 _step‚431 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4333861645824 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚41647250 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚11900519 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 215157760 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚41647252 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚41647252 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚27957668 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚402773040924800 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 trainer/global_step‚5249 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4333772978688 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚407584418847232 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚94476242 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚407095034508430 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚39858373 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚309729653842944 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4333772978688 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚305268215483904 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚52828992 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 121048064 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4461223201280 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚186 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚411918280493056 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚39858187 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚407584401807872 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚27957672 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚94486401 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚309729438685184¯úÕ=Æ2µ- _wandbruntime‚3645  _runtime‚3645s˜sï¶ 2loss/SRqqt2/assignment_loss‚0.5974042159088406 0loss/SRqqt1/detection_loss‚1.163422349136041 /loss/FBt1/detection_loss‚0.5227238239896552 &loss/total_loss‚10.084048015260255 _runtime‚3645.516763205 _step‚432 /loss/FRt1/assignment_loss‚2.408255155133729 1loss/FBt1/assignment_loss‚0.03428693508221844 1loss/FBt2/assignment_loss‚0.11573609883834318 epoch‚13 2loss/SRqqt1/assignment_loss‚0.6985605950271633 /loss/FRt1/detection_loss‚0.5971815479016407 /loss/FRt2/detection_loss‚1.3823651380544155 1loss/SRqqt2/detection_loss‚0.6701617501682052 /loss/FBt2/detection_loss‚0.9435964989952925 trainer/global_step‚5249 0loss/FRt2/assignment_loss‚0.9503539070247111 % _timestamp‚1.7844639711286323e+09î/á ·† trainer/global_step‚5249 0loss/FRt2/assignment_loss‚0.9503539070247111 % _timestamp‚1.7844639711286323e+09 2loss/SRqqt2/assignment_loss‚0.5974042159088406 0loss/SRqqt1/detection_loss‚1.163422349136041 /loss/FBt1/detection_loss‚0.5227238239896552 &loss/total_loss‚10.084048015260255 _runtime‚3645.516763205 _step‚432 /loss/FRt1/assignment_loss‚2.408255155133729 1loss/FBt1/assignment_loss‚0.03428693508221844 1loss/FBt2/assignment_loss‚0.11573609883834318 epoch‚13 2loss/SRqqt1/assignment_loss‚0.6985605950271633 /loss/FRt1/detection_loss‚0.5971815479016407 /loss/FRt2/detection_loss‚1.3823651380544155 1loss/SRqqt2/detection_loss‚0.6701617501682052 /loss/FBt2/detection_loss‚0.9435964989952925°‘ü÷x¸‚(:í ñ„óÒ¨§’¥& disk.nvme0n1.out‚38605.28271484375 disk.md0.in‚0 disk.md0.out‚ 89.53515625" disk.dm-1.out‚2000.17529296875$ memory_percent‚99.61998462677002) disk./.usagePercent‚76.34738312736627( disk.nvme0c0n1.out‚38605.28271484375' proc.memory.availableMB‚ 62.26171875# proc.memory.rssMB‚ 2913.99609375 network.sent‚ 7.856798e+06" disk.nvme0c0n1.in‚ 1852.4453125 disk.nvme1n1.in‚ 129.765625) proc.memory.percent‚17.78562068939209 cpu‚25.90926168203301 proc.cpu.threads‚17 disk.rbd0.in‚0 disk.nvme0n1.in‚ 1852.4453125 disk.nvme1c1n1.in‚ 129.765625( disk.nvme1c1n1.out‚16627.55615234375& disk.nvme1n1.out‚16627.55615234375 disk.dm-0.in‚ 1982.2109375# disk.dm-0.out‚53039.80810546875 disk.dm-1.in‚0 network.recv‚977211% disk./.usageGB‚1172.1231994628906 disk.rbd0.out‚0Uôž¹‚(:“ ñ„óÒý…Á¥ gpu.0.gpu‚88.0 gpu.0.memory‚21, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚48.0 gpu.0.powerWatts‚114.802' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚38.26733333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0%í ¢kº‚(:à €…óÒ¶¾ë¤ disk.md0.in‚0 cpu‚25.90787292258284 proc.cpu.threads‚17 network.recv‚981721 disk.dm-1.in‚0 disk.rbd0.out‚0& disk.nvme0c0n1.out‚38667.244140625 disk.nvme0n1.in‚ 1852.4453125 disk.md0.out‚89.8125# proc.memory.rssMB‚ 2913.99609375) proc.memory.percent‚17.78562068939209% disk./.usageGB‚1171.1753540039062 disk.dm-1.out‚2000.337890625" disk.nvme0c0n1.in‚ 1852.4453125$ memory_percent‚99.62003231048584' proc.memory.availableMB‚ 62.25390625 network.sent‚ 7.907042e+06) disk./.usagePercent‚76.28564429271495$ disk.nvme0n1.out‚38667.244140625 disk.nvme1c1n1.in‚ 129.765625% disk.nvme1n1.out‚16628.1630859375 disk.dm-0.in‚ 1982.2109375" disk.dm-0.out‚53101.5986328125 disk.rbd0.in‚0' disk.nvme1c1n1.out‚16628.1630859375 disk.nvme1n1.in‚ 129.765625¥ø×ü»‚(:’ €…óÒôÁ°¥ gpu.0.gpu‚36.0 gpu.0.memory‚8, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚50.0 gpu.0.powerWatts‚219.068' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚73.02266666666667 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ÇDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚415639178359296 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4501055005184 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚53312024 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚40211768 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚411266752496128 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚410773621462948 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4372425863168 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4360431580366 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚411266735456768 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚312520240617472 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚312520019161088 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4501182351872 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 151689728 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚415639072659968 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚28205412 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4360344793940 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚42021071 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚40211953 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚411266735456768 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 _runtime‚3678.859666569 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚53322167 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚415639072659968 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚12006537 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4372425863168 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚185 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚406413293708368 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 _step‚434 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4372337203200 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚95333095 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚308018964155904 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 127346688 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 % _timestamp‚1.7844640044705296e+09 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚12006356 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 trainer/global_step‚5299 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚415639178359296 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚308019058265600 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 221456384 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175½Œ^ÚßÁáO =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚95333095 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚308018964155904 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 127346688 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 % _timestamp‚1.7844640044705296e+09 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚12006356 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 trainer/global_step‚5299 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚415639178359296 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚308019058265600 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 221456384 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚28205416 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚181 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚406413276669008 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚42021073 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚415639178359296 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4501055005184 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚53312024 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚40211768 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚411266752496128 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚410773621462948 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4372425863168 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4360431580366 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚411266735456768 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚312520240617472 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚312520019161088 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4501182351872 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 151689728 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚415639072659968 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚28205412 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4360344793940 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚42021071 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚40211953 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚411266735456768 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 _runtime‚3678.859666569 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚53322167 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚415639072659968 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚12006537 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4372425863168 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚185 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚406413293708368 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 _step‚434 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4372337203200 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130²­9+2Â- _wandbruntime‚3679  _runtime‚3679ª‰2Øì&ÃæM aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4501991529472 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4361128288540 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚12008678 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12008488 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4361215075078 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚95360703 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚411341902162432 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4373122918912 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚42028702 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚28210472 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚312577187523072 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4501859995648 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚42028700 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚53332001 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚28210468 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 _runtime‚3679.281074679 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚42028702 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚53321844 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚415715113748480 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚415715008041984 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚95360703 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚308075195993600 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚415715113748480 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚95350544 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 131533824 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚53321844 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4373122918912 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚411341885123072 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚312576961879552 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 225643520 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚95350544 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚40218956 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 trainer/global_step‚5299 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚53332001 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚410848799308838 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4373211586048 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚411341885123072 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚411341902162432 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 151689728 _step‚435 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚194 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚410848695482940 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚406487584233760 % _timestamp‚1.7844640048924534e+09 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4373211586048 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840¤:±©ñ&ÄëM XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚312576961879552 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 225643520 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚95350544 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚40218956 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 trainer/global_step‚5299 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚53332001 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚410848799308838 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4373211586048 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚411341885123072 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚411341902162432 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 151689728 _step‚435 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚194 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚410848695482940 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚406487584233760 % _timestamp‚1.7844640048924534e+09 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4373211586048 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4501991529472 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4361128288540 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚12008678 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12008488 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4361215075078 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚95360703 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚411341902162432 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4373122918912 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚42028702 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚28210472 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚312577187523072 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4501859995648 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚42028700 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚53332001 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚28210468 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 _runtime‚3679.281074679 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚42028702 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚53321844 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚415715113748480 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚415715008041984 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚95360703 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚308075195993600 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚415715113748480 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚95350544 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 131533824 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚53321844 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4373122918912 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚411341885123072 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488³ëlS12Å- _wandbruntime‚3679  _runtime‚3679Ó«'4Æ‚ 1loss/FBt2/assignment_loss‚0.07374578561834864 1loss/SRqqt1/detection_loss‚0.9907685087688652 /loss/FBt2/detection_loss‚1.0358858775524866 trainer/global_step‚5299 $ _timestamp‚1.784464004893646e+09 _runtime‚3679.284809175 /loss/FRt1/assignment_loss‚2.342862594801226 2loss/SRqqt2/assignment_loss‚0.6224115665291603 0loss/FBt1/detection_loss‚0.44374690305975156 epoch‚13 0loss/FRt2/assignment_loss‚1.0988530652738189 2loss/SRqqt1/assignment_loss‚0.7597557563788322 1loss/FBt1/assignment_loss‚0.08631979286343514 /loss/FRt1/detection_loss‚0.5470399749053629 1loss/SRqqt2/detection_loss‚0.6862057308532082 &loss/total_loss‚10.183142308555674 _step‚436 /loss/FRt2/detection_loss‚1.4955467519511787"ùC° LJ 0loss/FBt1/detection_loss‚0.44374690305975156 epoch‚13 0loss/FRt2/assignment_loss‚1.0988530652738189 2loss/SRqqt1/assignment_loss‚0.7597557563788322 1loss/FBt1/assignment_loss‚0.08631979286343514 /loss/FRt1/detection_loss‚0.5470399749053629 1loss/SRqqt2/detection_loss‚0.6862057308532082 &loss/total_loss‚10.183142308555674 _step‚436 /loss/FRt2/detection_loss‚1.4955467519511787 1loss/FBt2/assignment_loss‚0.07374578561834864 1loss/SRqqt1/detection_loss‚0.9907685087688652 /loss/FBt2/detection_loss‚1.0358858775524866 trainer/global_step‚5299 $ _timestamp‚1.784464004893646e+09 _runtime‚3679.284809175 /loss/FRt1/assignment_loss‚2.342862594801226 2loss/SRqqt2/assignment_loss‚0.6224115665291603´Ô 7$qÈ‚(:æ …óÒÃûÔ¤ network.recv‚984822 disk.rbd0.in‚0 disk.rbd0.out‚0" disk.nvme0c0n1.in‚ 1852.4453125( disk.nvme0c0n1.out‚38678.26904296875 disk.nvme0n1.in‚ 1852.4453125 disk.dm-0.in‚ 1982.2109375 disk.dm-1.in‚0 network.sent‚ 7.912967e+06# disk./.usageGB‚1171.17533493042' disk.nvme1c1n1.out‚16630.6904296875 disk.nvme1n1.in‚ 129.765625 disk.md0.in‚0$ memory_percent‚99.62031841278076) proc.memory.percent‚17.78562068939209 cpu‚25.906493601462174) disk./.usagePercent‚76.28564305034489 disk.nvme1c1n1.in‚ 129.765625# proc.memory.rssMB‚ 2913.99609375 proc.cpu.threads‚17& disk.nvme0n1.out‚38678.26904296875% disk.nvme1n1.out‚16630.6904296875 disk.md0.out‚ 90.109375# disk.dm-0.out‚53114.39697265625 disk.dm-1.out‚2000.451171875' proc.memory.availableMB‚ 62.20703125Wø–ÀÉ‚(:’ …óÒºµÀ¥ gpu.0.gpu‚35.0 gpu.0.memory‚8, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚49.0 gpu.0.powerWatts‚150.092' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚50.03066666666667 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0þ}í“Ê‚(:ˆ ž…óÒ›áÞ¤ gpu.0.gpu‚90.0 gpu.0.memory‚18, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚50.0 gpu.0.powerWatts‚214.497' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚71.499 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0V_¼^vË‚(:ë ž…óÒ¼ù€¥( disk.nvme0c0n1.out‚38678.93017578125& disk.nvme0n1.out‚38678.93017578125& disk.nvme1n1.out‚16631.21630859375 disk.md0.in‚0$ disk./.usageGB‚1171.174617767334 disk.dm-1.in‚0 disk.nvme1c1n1.in‚ 129.765625( disk.nvme1c1n1.out‚16631.21630859375 disk.md0.out‚ 90.34765625$ memory_percent‚99.62830543518066# proc.memory.rssMB‚ 2913.99609375) proc.memory.percent‚17.78562068939209 network.recv‚989456 disk.rbd0.out‚0 disk.nvme1n1.in‚ 129.765625 cpu‚25.90612618133998 network.sent‚ 7.976289e+06) disk./.usagePercent‚76.28559633723067 disk.rbd0.in‚0 disk.nvme0n1.in‚ 1852.4453125& proc.memory.availableMB‚ 60.8984375 proc.cpu.threads‚17 disk.dm-0.in‚ 1982.2109375# disk.dm-0.out‚53114.94775390625" disk.dm-1.out‚2000.55419921875" disk.nvme0c0n1.in‚ 1852.4453125¡ÀÁ 2Ì- _wandbruntime‚3712  _runtime‚3712ø`Ù©ÈÍ _step‚437 ' lr-AdamW/pg1‚0.00011846282186691763 ' lr-AdamW/pg2‚0.00011846282186691763 trainer/global_step‚5349 $ _timestamp‚1.784464037988369e+09 _runtime‚3712.377834202Ò.äÍÎÇ _step‚437 ' lr-AdamW/pg1‚0.00011846282186691763 ' lr-AdamW/pg2‚0.00011846282186691763 trainer/global_step‚5349 $ _timestamp‚1.784464037988369e+09 _runtime‚3712.377834202µÕ·XF2Ï- _wandbruntime‚3712  _runtime‚3712 è[À"ÐÛO WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚96207397 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚28458212 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚414527490089142 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚53815019 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 151689728 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚40571863 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚28458216 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚419436011614720 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚419435905915392 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 ^LDeviceStatsMoÚ±½"¿nitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4411775803392 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚42402523 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚310825850555904 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4541733802496 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚414527386263356 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚410127802938608 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4411687143424 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚410127819977968 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚40572034 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚415024218771968 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 $ _timestamp‚1.784464037989216e+09 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4541600164352 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4399583324748 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚414527490089142 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚53815019 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 151689728 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚40571863 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚28458216 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚419436011614720 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚419435905915392 IDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4411775803392 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚42402523 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚310825850555904 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4541733802496 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚414527386263356 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚410127802938608 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4411687143424 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚410127819977968 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚40572034 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚415024218771968 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 $ _timestamp‚1.784464037989216e+09 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4541600164352 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4399583324748 I6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚28463268 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚415099385477632 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 121048064 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚166 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 % _timestamp‚1.7844640384024832e+09 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚419511841297408 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚96224846 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚53814696 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚315424612553216 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚410202110503360 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚53814696 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚96235005 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚96235005 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚42410150 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚162 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 151689728 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚42410150 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4412561526272 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚96224846 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚28463272 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚53824853 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4412561526272 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚419511947003904 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12115784 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚42410152 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚419511947003904 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚315424397395456 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 trainer/global_step‚5349 _runtime‚3712.791068266 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚40579052 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4542409111552 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4412472859136 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚40579218 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚415099368438272 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚415099368438272ÏnëÔëM ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚410202110503360 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚53814696 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚96235005 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚96235005 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚42410150 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚162 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 151689728 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚42410150 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4412561526272 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚96224846 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚28463272 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚53824853 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4412561526272 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚419511947003904 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12115784 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚42410152 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚419511947003904 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚315424397395456 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 trainer/global_step‚5349 _runtime‚3712.791068266 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚40579052 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4542409111552 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4412472859136 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚40579218 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚415099368438272 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 ZFDevicÄœƒ?×eStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚415099368438272 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚28463268 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚415099385477632 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 121048064 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 FDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 EDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚166 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 % _timestamp‚1.7844640384024832e+09 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚419511841297408 P6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚96224846 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚53814696 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚315424612553216·Ñ^Áº2Õ- _wandbruntime‚3712  _runtime‚3712'Nã:Ö 0loss/FBt2/assignment_loss‚0.1560376227485344 1loss/SRqqt2/detection_loss‚0.6635544999442214 0loss/FBt1/detection_loss‚0.37712287692329793 /loss/FRt1/assignment_loss‚2.387816839339413 /loss/FRt2/assignment_loss‚1.097935537760109 2loss/SRqqt1/assignment_loss‚0.6032127775947334 1loss/FBt1/assignment_loss‚0.03519538439130731 % _timestamp‚1.7844640384035542e+09 /loss/FRt1/detection_loss‚0.5744259536106445 /loss/FBt2/detection_loss‚1.0747450130203595 &loss/total_loss‚10.138869733739268 epoch‚13 _runtime‚3712.791635237 _step‚440 2loss/SRqqt2/assignment_loss‚0.6354169690673921 /loss/FRt2/detection_loss‚1.4117505830551527 1loss/SRqqt1/detection_loss‚1.1216556762841035 trainer/global_step‚5349$¼fž ׆ 0loss/FBt2/assignment_loss‚0.1560376227485344 1loss/SRqqt2/detection_loss‚0.6635544999442214 0loss/FBt1/detection_loss‚0.37712287692329793 /loss/FRt1/assignment_loss‚2.387816839339413 /loss/FRt2/assignment_loss‚1.097935537760109 2loss/SRqqt1/assignment_loss‚0.6032127775947334 1loss/FBt1/assignment_loss‚0.03519538439130731 % _timestamp‚1.7844640384035542e+09 /loss/FRt1/detection_loss‚0.5744259536106445 /loss/FBt2/detection_loss‚1.0747450130203595 &loss/total_loss‚10.138869733739268 epoch‚13 _runtime‚3712.791635237 _step‚440 2loss/SRqqt2/assignment_loss‚0.6354169690673921 /loss/FRt2/detection_loss‚1.4117505830551527 1loss/SRqqt1/detection_loss‚1.1216556762841035 trainer/global_step‚5349¸v5ZxØ‚(:í ­…óÒÃó„¥ disk.rbd0.in‚0 disk.rbd0.out‚0 disk.md0.in‚0 disk.md0.out‚ 90.74609375 disk.dm-1.in‚0 network.recv‚992497) disk./.usagePercent‚76.30608227407868" disk.nvme0c0n1.in‚ 1852.4453125 disk.nvme1c1n1.in‚ 129.765625( disk.nvme1c1n1.out‚16655.93408203125# disk.dm-0.out‚53254.44873046875$ memory_percent‚99.62728023529053 network.sent‚ 7.992054e+06$ disk./.usageGB‚1171.489128112793( disk.nvme0c0n1.out‚38795.03955078125 disk.nvme0n1.in‚ 1852.4453125& disk.nvme0n1.out‚38795.03955078125 disk.dm-0.in‚ 1982.2109375" disk.dm-1.out‚2001.03857421875# proc.memory.rssMB‚ 2913.99609375 disk.nvme1n1.in‚ 129.765625& disk.nvme1n1.out‚16655.93408203125' proc.memory.availableMB‚ 61.06640625) proc.memory.percent‚17.78562068939209 cpu‚25.905833252128378 proc.cpu.threads‚17˜«l›Ù‚(: ­…óÒ¢ò¹¥ gpu.0.gpu‚0.0 gpu.0.memory‚0, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚49.0 gpu.0.powerWatts‚223.33' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚74.44333333333334 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0&ÍÓÀrÚ‚(:ç ¼…óÒº—´¤ disk.rbd0.out‚0" disk.nvme0c0n1.in‚ 1852.4453125( disk.nvme0c0n1.out‚39813.32568359375' disk.nvme1c1n1.out‚16663.0673828125# disk.dm-0.out‚54278.73388671875) disk./.usagePercent‚76.35942939593066 disk.dm-1.in‚0# proc.memory.rssMB‚ 2913.99609375 cpu‚25.904418479299917% disk./.usageGB‚1172.3081398010254! disk.dm-1.out‚2001.2333984375& disk.nvme0n1.out‚39813.32568359375 disk.nvme1c1n1.in‚ 129.765625 disk.nvme1n1.in‚ 129.765625 disk.md0.in‚0$ memory_percent‚99.62785243988037 disk.nvme0n1.in‚ 1852.4453125% disk.nvme1n1.out‚16663.0673828125 disk.md0.out‚91.1875 disk.dm-0.in‚ 1982.2109375' proc.memory.availableMB‚ 60.97265625) proc.memory.percent‚17.78562068939209 proc.cpu.threads‚17 network.sent‚ 8.041551e+06 network.recv‚996601 disk.rbd0.in‚0£ñÔžÛ‚(:“ ¼…óÒÙÉ‚¥ gpu.0.gpu‚67.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚50.0 gpu.0.powerWatts‚224.848' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚74.94933333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Œ\ 2Ü- _wandbruntime‚3745  _runtime‚3745ã%%<ÉÝà _step‚441 ' lr-AdamW/pg1‚0.00011806363123594537 ' lr-AdamW/pg2‚0.00011806363123594537 trainer/global_step‚5399 % _timestamp‚1.7844640708187852e+09 _runtime‚3745.2078388483lÎÞÈ trainer/global_step‚5399 % _timestamp‚1.7844640708187852e+09 _runtime‚3745.207838848 _step‚441 ' lr-AdamW/pg1‚0.00011806363123594537 ' lr-AdamW/pg2‚0.00011806363123594537¹ø+“Q2ß- _wandbruntime‚3745  _runtime‚3745¦[œ-â'àÜO TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4451037083648 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚423232739170816 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚423232739170816 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚54297728 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚418281254889550 % _timestamp‚1.7844640708195052e+09 _step‚442 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚40931624 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4451125743616 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4582236034560 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚418781702087168 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚42783971 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4438908641982 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚418781702087168 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4451037083648 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚28711012 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚42783973 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚413842346247568 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 trainer/global_step‚5399 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚97081699 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚54307871 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚54307871 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚318214854032384 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚418781719126528 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚423232844870144 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚42783973 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚313632831065600 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚413842329208208 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚28711016 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4438821855556 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚318215067100160 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚313632736955904 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚418781719126528 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 213067776 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4451125743616 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚169 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚418281151063764 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚54297728 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 151689728 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚165 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚423232739170816 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚54297728 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚418281254889550 % _timestamp‚1.7844640708195052e+09 _step‚442 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚40931624 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4451125743616 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4582236034560 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚418781702087168 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚42783971 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4438908641982 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚418781702087168 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4451037083648 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚28711012 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚42783973 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚413842346247568 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 trainer/global_step‚5399 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚97081699 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚54307871 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚54307871 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚318214854032384 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚418781719126528 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚423232844870144 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚42783973 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚313632831065600 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚413842329208208 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚28711016 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4438821855556 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚318215067100160 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚313632736955904 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚418781719126528 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 213067776 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4451125743616 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚169 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚418281151063764 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚54297728 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 151689728 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚165 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 IDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚423232739170816 BDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4451911466496 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4451822799360 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚97099148 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚313688874683904 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚413916619733600 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4451911466496 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 _runtime‚3745.624547288 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚40938842 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 123145216 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚418356225083756 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚418856851753472 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4582922746880 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 % _timestamp‚1.7844640712359934e+09 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚418856868792832 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚97109307 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚318272014685696 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 217254912 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚423308780259328 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4439605350156 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4583045892096 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚54307548 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚318271797430784 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4451822799360 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚40939010 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 trainer/global_step‚5399 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4439692136694 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚97109307 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚97099148 PDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚97109307 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚97099148 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4451911466496 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4451822799360 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚97099148 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚313688874683904 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚413916619733600 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4451911466496 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 _runtime‚3745.624547288 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 FDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚40938842 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 123145216 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚418356225083756 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚418856851753472 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4582922746880 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 % _timestamp‚1.7844640712359934e+09 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚418856868792832 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚97109307 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚318272014685696 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 217254912 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚423308780259328 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4439605350156 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4583045892096 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚54307548 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360»G|G2å- _wandbruntime‚3745  _runtime‚3745V»Íæ‚ 2loss/SRqqt2/assignment_loss‚0.5864740992190001 /loss/FRt1/detection_loss‚0.5498001607374475 1loss/SRqqt1/detection_loss‚1.0409287618602947 1loss/SRqqt2/detection_loss‚0.6805656380620254 %loss/total_loss‚10.49231392071837 0loss/FRt1/assignment_loss‚2.4373430759491588 0loss/FRt2/assignment_loss‚0.9786899160932411 2loss/FBt1/assignment_loss‚0.054703868894081145 epoch‚13 trainer/global_step‚5399 % _timestamp‚1.7844640712377708e+09 _step‚444 2loss/SRqqt1/assignment_loss‚0.7073027004154311 /loss/FRt2/detection_loss‚1.4474656282858853 .loss/FBt1/detection_loss‚0.506850555433684 /loss/FBt2/detection_loss‚1.4871786627239392 _runtime‚ 3745.62602448 2loss/FBt2/assignment_loss‚0.015010853044183628:-6z ç‡ 2loss/SRqqt2/assignment_loss‚0.5864740992190001 /loss/FRt1/detection_loss‚0.5498001607374475 1loss/SRqqt1/detection_loss‚1.0409287618602947 1loss/SRqqt2/detection_loss‚0.6805656380620254 %loss/total_loss‚10.49231392071837 0loss/FRt1/assignment_loss‚2.4373430759491588 0loss/FRt2/assignment_loss‚0.9786899160932411 2loss/FBt1/assignment_loss‚0.054703868894081145 epoch‚13 trainer/global_step‚5399 % _timestamp‚1.7844640712377708e+09 _step‚444 2loss/SRqqt1/assignment_loss‚0.7073027004154311 /loss/FRt2/detection_loss‚1.4474656282858853 .loss/FBt1/detection_loss‚0.506850555433684 /loss/FBt2/detection_loss‚1.4871786627239392 _runtime‚ 3745.62602448 2loss/FBt2/assignment_loss‚0.015010853044183628¼–µxhè‚(:Ý Ë…óÒÓͤ! disk.dm-1.out‚2001.4716796875 disk.rbd0.in‚0' disk.nvme0c0n1.out‚39816.2763671875 disk.nvme1c1n1.in‚ 129.765625& proc.memory.availableMB‚ 60.1796875# proc.memory.rssMB‚ 2913.99609375 cpu‚25.903137073269594( disk./.usagePercent‚76.3596413442627! disk.dm-0.out‚54281.580078125 disk.dm-1.in‚0" disk.nvme0c0n1.in‚ 1852.4453125 disk.nvme0n1.in‚ 1852.4453125% disk.nvme0n1.out‚39816.2763671875 network.recv‚999764$ disk./.usageGB‚1172.311393737793 disk.nvme1n1.in‚ 129.765625# disk.nvme1n1.out‚16663.89453125 disk.md0.out‚ 91.515625$ memory_percent‚99.63269233703613 proc.cpu.threads‚17 network.sent‚ 8.04748e+06 disk.md0.in‚0 disk.rbd0.out‚0% disk.nvme1c1n1.out‚16663.89453125) proc.memory.percent‚17.78562068939209 disk.dm-0.in‚ 1982.2109375Ç~}Vé‚(:’ Ë…óÒØšÿ¤ gpu.0.gpu‚42.0 gpu.0.memory‚9, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚50.0 gpu.0.powerWatts‚224.956' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚74.98533333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¬™Ätê‚(:é Ú…óÒòÐå¤% disk./.usageGB‚1172.3114013671875 disk.md0.in‚0 disk.dm-0.in‚ 1982.2109375 proc.cpu.threads‚17# disk.dm-0.out‚54282.46533203125 disk.dm-1.in‚0 disk.rbd0.in‚0 disk.rbd0.out‚0" disk.nvme0c0n1.in‚ 1852.4453125 disk.nvme0n1.in‚ 1852.4453125' proc.memory.availableMB‚ 60.04296875# proc.memory.rssMB‚ 2913.99609375 network.sent‚ 8.110653e+06 disk.dm-1.out‚ 2001.609375( disk.nvme0c0n1.out‚39817.33837890625& disk.nvme0n1.out‚39817.33837890625 disk.nvme1c1n1.in‚ 129.765625& disk.nvme1c1n1.out‚16664.513671875) proc.memory.percent‚17.78562068939209 cpu‚25.90186357422277) disk./.usagePercent‚76.35964184121072 disk.nvme1n1.in‚ 129.765625$ disk.nvme1n1.out‚16664.513671875 disk.md0.out‚ 91.8203125$ memory_percent‚99.63352680206299 network.recv‚ 1.004023e+06ëò¾ ë‚(:’ Ú…óÒ¹úš¥ gpu.0.gpu‚90.0 gpu.0.memory‚18, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚49.0 gpu.0.powerWatts‚123.68' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚41.22666666666667 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0z•hñ2ì- _wandbruntime‚3778  _runtime‚3778Æ)ãÉíà ' lr-AdamW/pg1‚0.00011766132924425825 ' lr-AdamW/pg2‚0.00011766132924425825 trainer/global_step‚5449 % _timestamp‚1.7844641038778124e+09 _runtime‚3778.266217225 _step‚445„4ƒÎîÈ % _timestamp‚1.7844641038778124e+09 _runtime‚3778.266217225 _step‚445 ' lr-AdamW/pg1‚0.00011766132924425825 ' lr-AdamW/pg2‚0.00011766132924425825 trainer/global_step‚5449½âñ»2ï- _wandbruntime‚3778  _runtime‚3778S,d*â'ðÜO \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚422539185402368 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4478147172790 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚54800723 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚28963816 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4478060386364 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚43165421 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚43165423 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚97966146 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚43165421 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4622770160640 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚422035019689958 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚54800723 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚186 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚12326061 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4490475683840 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚427029572426240 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 trainer/global_step‚5449 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚28963812 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4490475683840 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚41290063 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚190 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚12326247 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 135735296 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4622634425344 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 151689728 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚422539202441728 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚422034915864172 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚417556872517168 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚427029572426240 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚417556855477808 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4490387023872 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚316439623355904 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 _runtime‚3778.267259347 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4490387023872 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚427029678125568õî\OÛñáO ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4490475683840 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚41290063 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚190 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚12326247 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 135735296 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4622634425344 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 151689728 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚422539202441728 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚422034915864172 `LDeviceStatsMonitor.on_train_batch_sG1ô "tart/requested_bytes.large_pool.allocated‚417556872517168 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚427029572426240 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚417556855477808 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4490387023872 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚316439623355904 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 _runtime‚3778.267259347 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4490387023872 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚427029678125568 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚422539185402368 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4478147172790 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚54800723 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚28963816 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4478060386364 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚43165421 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚43165423 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚97966146 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚43165421 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4622770160640 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚422035019689958 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚54800723 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚186 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚12326061 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4490475683840 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚427029572426240 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 trainer/global_step‚5449 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚28963812 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682¾ñÊ]2ò- _wandbruntime‚3778  _runtime‚3778ó •ë&óåM YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4491261406720 $ _timestamp‚1.784464104292423e+09 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚97973450 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚54810557 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚12328273 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚97983609 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4478843880964 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚54800400 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚54800400 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚28968872 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚43173050 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 225643520 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚97973450 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚422614352108032 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚28968868 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚321119426274304 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚316495855193600 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚168 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚97983609 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚164 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚417631163042560 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 trainer/global_step‚5449 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚41296977 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚427105507808256 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4491261406720 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚316495761083904 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚422109989884164 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 PDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12328109 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 131533824 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚43173052 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚321119200630784 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚417631146003200 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4478930667502 _step‚447 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4491172739584 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0šu‚!ð&ôêM NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚43173052 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚422614335068672 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12328109 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 131533824 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚43173052 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚321119200630784 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚417631146003200 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4478930667502 _step‚447 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4491172739584 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4491261406720 $ _timestamp‚1.784464104292423e+09 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚97973450 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚54810557 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚12328273 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚97983609 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4478843880964 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚54800400 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚54800400 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 EDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚28968872 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚43173050 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 225643520 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚97973450 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚422614352108032 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚28968868 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚321119426274304 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚316495855193600 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚168 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚97983609 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚164 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚417631163042560 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 trainer/global_step‚5449 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚41296977 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚427105507808256 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4491261406720 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚316495761083904 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚422109989884164 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 PDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚430826405681664 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4517298917172 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚426296685756928 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚178 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚43546873 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4517385703598 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚55293575 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚29216612 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚174 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚55283432 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚430826405681664 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚430826511380992 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 229844992 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚323909667073024 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚426296668717568 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚12432500 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4663157317120 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 % _timestamp‚1.7844641364042518e+09 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚29216616 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 151689728 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4529736964096 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚55283432 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚430826511380992 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 229844992 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚323909667073024 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚426296668717568 BDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚12432500 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4663157317120 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 % _timestamp‚1.7844641364042518e+09 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚29216616 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 151689728 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4529736964096 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚55283432 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 IDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚430826405681664 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4517298917172 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚426296685756928 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚178 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚43546873 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4517385703598 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚55293575 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚29216612 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚174 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130ÂYy~2„- _wandbruntime‚3811  _runtime‚3811æ"?ì&…æM [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 125242368 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚425863858510470 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4530522679808 _runtime‚3811.203216207 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚98857911 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚12434853 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 219352064 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚421345672272800 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚29221668 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚426371835423232 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4663966919680 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚29221672 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚182 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚319302647483904 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚421345689312160 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚43554502 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚41656339 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚425863754684572 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚98857911 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 trainer/global_step‚5499 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚430902446770176 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚426371818383872 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚98847752 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚186 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚43554502 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 % _timestamp‚1.7844641368144634e+09 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12434671 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚323966833755648 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚323966614403584 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 _step‚451 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚430902446770176 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 151689728 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4518082411772 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚55293252 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚319302741593600 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚55303409 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚41656525 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚426371835423232 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚43554500 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚55303409 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360ÍcÉ÷†ëM 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 _step‚451 N>DeviceStatsMo‘j†3ú%nitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚430902446770176 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 151689728 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4518082411772 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚55293252 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚319302741593600 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚55303409 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚41656525 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚426371835423232 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚43554500 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚55303409 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 125242368 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚425863858510470 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4530522679808 _runtime‚3811.203216207 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚98857911 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚12434853 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 219352064 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚421345672272800 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚29221668 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚426371835423232 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4663966919680 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚29221672 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚182 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚319302647483904 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚421345689312160 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚43554502 EDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚41656339 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚425863754684572 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚98857911 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 trainer/global_step‚5499 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚430902446770176 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚426371818383872 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚98847752 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚186 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚43554502 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 % _timestamp‚1.7844641368144634e+09 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12434671 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚323966833755648 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚323966614403584 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0Ã-42‡- _wandbruntime‚3811  _runtime‚3811í®[/ ˆƒ 2loss/SRqqt1/assignment_loss‚0.5800654552189994 /loss/FRt1/detection_loss‚0.5605554673089574 0loss/FRt2/assignment_loss‚1.0983928127297715 1loss/SRqqt2/detection_loss‚0.6354780145169554 0loss/FBt1/detection_loss‚0.49086535088773703 _runtime‚3811.203760718 2loss/SRqqt2/assignment_loss‚0.6617904183611585 2loss/FBt1/assignment_loss‚0.023182035843872288 /loss/FBt2/detection_loss‚0.8581981201966603 %loss/total_loss‚9.925339945745446 % _timestamp‚1.7844641368154914e+09 /loss/FRt1/assignment_loss‚2.373427184689413 1loss/FBt2/assignment_loss‚0.07987217761086261 /loss/FRt2/detection_loss‚1.4952489743609612 1loss/SRqqt1/detection_loss‚1.0682639340200957 epoch‚13 trainer/global_step‚5499 _step‚452ÏÑ“‰ˆ 0loss/FRt2/assignment_loss‚1.0983928127297715 1loss/SRqqt2/detection_loss‚0.6354780145169554 0loss/FBt1/detection_loss‚0.49086535088773703 _runtime‚3811.203760718 2loss/SRqqt2/assignment_loss‚0.6617904183611585 2loss/FBt1/assignment_loss‚0.023182035843872288 /loss/FBt2/detection_loss‚0.8581981201966603 %loss/total_loss‚9.925339945745446 % _timestamp‚1.7844641368154914e+09 /loss/FRt1/assignment_loss‚2.373427184689413 1loss/FBt2/assignment_loss‚0.07987217761086261 /loss/FRt2/detection_loss‚1.4952489743609612 1loss/SRqqt1/detection_loss‚1.0682639340200957 epoch‚13 trainer/global_step‚5499 _step‚452 2loss/SRqqt1/assignment_loss‚0.5800654552189994 /loss/FRt1/detection_loss‚0.5605554673089574Ä‚×ïCwŠ‚(:ì –†óÒï¡‹¥ disk.dm-1.in‚0 disk.dm-0.out‚54311.12109375& proc.memory.availableMB‚ 61.6640625# proc.memory.rssMB‚ 2913.99609375 proc.cpu.threads‚17( disk.nvme1c1n1.out‚16671.15185546875% disk./.usageGB‚1172.0730743408203" disk.dm-1.out‚2001.98876953125 disk.rbd0.out‚0 disk.nvme0n1.in‚ 1852.4453125 disk.dm-0.in‚ 1982.2109375 cpu‚25.896815256291177 disk.rbd0.in‚0% disk.nvme0n1.out‚39842.5498046875 disk.md0.out‚ 93.109375$ memory_percent‚99.62363243103027) proc.memory.percent‚17.78562068939209 disk.nvme1c1n1.in‚ 129.765625 network.sent‚ 8.198726e+06 network.recv‚ 1.018103e+06" disk.nvme0c0n1.in‚ 1852.4453125' disk.nvme0c0n1.out‚39842.5498046875 disk.nvme1n1.in‚ 129.765625& disk.nvme1n1.out‚16671.15185546875 disk.md0.in‚0) disk./.usagePercent‚76.34411817885167öÍÉ“‹‚(:ˆ –†óÒ‹ÏÒ¥ gpu.0.gpu‚47.0 gpu.0.memory‚11, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚50.0 gpu.0.powerWatts‚224.277' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚74.759 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0?VRLzŒ‚(:ï ¥†óÒÆÂú¤ disk.nvme1c1n1.in‚ 129.765625! disk.dm-0.out‚54312.685546875 disk.rbd0.in‚0 disk.rbd0.out‚0( disk.nvme0c0n1.out‚39844.33837890625) proc.memory.percent‚17.78562068939209 cpu‚25.89553193184885) disk./.usagePercent‚76.34414799573308 disk.nvme1n1.in‚ 129.765625% disk.nvme1n1.out‚16671.6181640625 disk.dm-0.in‚ 1982.2109375 disk.dm-1.in‚0$ memory_percent‚99.62055683135986' proc.memory.availableMB‚ 62.16796875 proc.cpu.threads‚17 network.sent‚ 8.250352e+06 network.recv‚ 1.022349e+06% disk./.usageGB‚1172.0735321044922 disk.md0.out‚ 93.40234375" disk.dm-1.out‚2002.03564453125" disk.nvme0c0n1.in‚ 1852.4453125 disk.nvme0n1.in‚ 1852.4453125& disk.nvme0n1.out‚39844.33837890625' disk.nvme1c1n1.out‚16671.6181640625 disk.md0.in‚0# proc.memory.rssMB‚ 2913.99609375¦ndž“‚(:ˆ ¥†óÒáꨥ gpu.0.gpu‚48.0 gpu.0.memory‚12, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚50.0 gpu.0.powerWatts‚159.183' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚53.061 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚00#OÜ2Ž- _wandbruntime‚3842  _runtime‚3842Ø?ïÆÀ ' lr-AdamW/pg1‚0.00011684749473333737 ' lr-AdamW/pg2‚0.00011684749473333737 trainer/global_step‚5549 $ _timestamp‚1.784464168505171e+09 _runtime‚ 3842.8935474 _step‚453‘úßËÅ $ _timestamp‚1.784464168505171e+09 _runtime‚ 3842.8935474 _step‚453 ' lr-AdamW/pg1‚0.00011684749473333737 ' lr-AdamW/pg2‚0.00011684749473333737 trainer/global_step‚5549ŨS}ÿ2‘- _wandbruntime‚3842  _runtime‚3842ôÜâ'’ÜO TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚29469412 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4569175564288 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚55786427 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚29469416 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚99704605 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚43928321 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚99714750 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚322053396155904 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚430054169072128 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 _step‚454 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚434623344636416 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚434623238937088 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚326757048255488 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4703652099584 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚429542445464988 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚322053490265600 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚55776284 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚434623344636416 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4556537447980 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 225650688 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚42007503 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚430054152032768 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚43928323 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 131540992 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚429542549290774 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4569086904320 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4569175564288 IDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚29469416 E8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚99704605 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚43928321 M@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚99714750 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚322053396155904 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚430054169072128 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 _step‚454 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚434623344636416 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚434623238937088 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚326757048255488 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4703652099584 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚429542445464988 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚322053490265600 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚55776284 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚434623344636416 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4556537447980 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 225650688 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚42007503 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚430054152032768 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚43928323 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 131540992 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚429542549290774 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4569086904320 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 OIDŽA:•zeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4569175564288 IDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303ÆäÁ¶22”- _wandbruntime‚3843  _runtime‚3843NŸ˜ì&•æM REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚29474468 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚322109627993600 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚99732213 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚99722054 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚434699174319104 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚43935950 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚43935952 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4704596652032 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚42014635 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 trainer/global_step‚5549 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚181 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚55786104 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4569961287168 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚434699280025600 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 _step‚455 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚430129318738432 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚185 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4557407729118 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚322109533883904 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚429617519484980 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚55796261 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 _runtime‚3843.306023337 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚55786104 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚43935952 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚430129318738432 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚99722054 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚29474472 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚425060198542400 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 227740672 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚434699280025600 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚12540163 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚99732213 % _timestamp‚1.7844641689174697e+09 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚326813996904960 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚42014450 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12539982 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚55796261 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚425060215581760 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 EDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4569872620032 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚430129301699072 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509H6¼Ìñ&–ëM L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4569872620032 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚430129301699072 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚29474468 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚322109627993600 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 G:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚99732213 C6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚99722054 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚434699174319104 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚43935950 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚43935952 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4704596652032 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚42014635 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 trainer/global_step‚5549 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚181 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚55786104 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4569961287168 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚434699280025600 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 _step‚455 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚430129318738432 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚185 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4557407729118 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚322109533883904 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚429617519484980 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚55796261 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 _runtime‚3843.306023337 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚55786104 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚43935952 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚430129318738432 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 G:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚99722054 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚29474472 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚425060198542400 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 227740672 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚434699280025600 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚12540163 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 K>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚99732213 % _timestamp‚1.7844641689174697e+09 P6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚326813996904960 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚42014450 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12539982 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚55796261 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚425060215581760 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 EDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 _step‚458 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4608436844544 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚433296314091182 trainer/global_step‚5599 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4608436844544 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4608525504512 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 % _timestamp‚1.7844642011191473e+09 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚433811635347968 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 100589052 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚12643498 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚329604347036672 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JËÚLytes.all.peak‚ 2372707840 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 133638144 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚433296210265396 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4595775978788 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 227747840 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚438420177891840 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4744198118912 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 151689728 _runtime‚3875.507397461 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚438420072192512 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚433811635347968 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚324860376665600 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚438420072192512 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚29722212 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚12643325 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚324860376665600 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚438420072192512 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚29722212 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚12643325 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 _step‚458 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4608436844544 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚433296314091182 trainer/global_step‚5599 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4608436844544 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4608525504512 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 % _timestamp‚1.7844642011191473e+09 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚433811635347968 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 100589052 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚12643498 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚329604347036672 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 JDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚433886785014272 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚433886802053632 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚433371388111286 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4609311227392 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚428774724812000 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚56289113 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 100596356 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4609222560256 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚29727272 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚42372949 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 229837824 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 100596356 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚56278956 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚438496113281024 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚56289113 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚42372780 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚165 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 100606515 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 _step‚459 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12645512 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚428774741851360 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4596559473388 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚438496007574528 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚433886785014272 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚169 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 $ _timestamp‚1.784464201530814e+09 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4745006510080 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4596646259926 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚44317400 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 135728128 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4744870781952 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 100606515 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚433886802053632 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚438496113281024 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚324916514393600 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚329661291065856 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚433371284285388 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 _runtime‚3875.919306455 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2›~6Ì…¦îM KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4609222560256 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚29727272 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚42372949 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 229837824 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 100596356 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚56278956 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚438496113281024 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚56289113 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚42372780 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚165 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 FDeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 100606515 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 _step‚459 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12645512 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚428774741851360 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4596559473388 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚438496007574528 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚433886785014272 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚169 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 $ _timestamp‚1.784464201530814e+09 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4745006510080 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4596646259926 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚44317400 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 135728128 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4744870781952 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 100606515 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚433886802053632 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚438496113281024 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚324916514393600 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚329661291065856 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚433371284285388 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 _runtime‚3875.919306455 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚433886785014272 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚433886802053632 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_poo• Tol.peak‚ 269300736 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚433371388111286 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4609311227392 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚428774724812000 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚56289113 PDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 100596356 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1Ë–=í72§- _wandbruntime‚3875  _runtime‚3875b '’ ¨ƒ /loss/FRt2/detection_loss‚1.4742937320701028 /loss/FBt2/detection_loss‚1.2149479350902967 _runtime‚3875.919965656 0loss/FRt1/assignment_loss‚2.3382782838424716 &loss/total_loss‚10.308360368932076 % _timestamp‚1.7844642015318272e+09 2loss/SRqqt1/assignment_loss‚0.8045330433433107 1loss/FBt1/assignment_loss‚0.03988520283752249 /loss/FRt1/detection_loss‚0.5704062783011336 1loss/SRqqt1/detection_loss‚0.9518337144111486 /loss/FBt1/detection_loss‚0.4407117082456669 trainer/global_step‚5599 _step‚460 2loss/SRqqt2/assignment_loss‚0.6286483715572048 2loss/FBt2/assignment_loss‚0.036580490762410725 1loss/SRqqt2/detection_loss‚0.7020666066183103 epoch‚13 /loss/FRt2/assignment_loss‚1.106175001852498P¸ˆÀ©ˆ 0loss/FRt1/assignment_loss‚2.3382782838424716 &loss/total_loss‚10.308360368932076 % _timestamp‚1.7844642015318272e+09 2loss/SRqqt1/assignment_loss‚0.8045330433433107 1loss/FBt1/assignment_loss‚0.03988520283752249 /loss/FRt1/detection_loss‚0.5704062783011336 1loss/SRqqt1/detection_loss‚0.9518337144111486 /loss/FBt1/detection_loss‚0.4407117082456669 trainer/global_step‚5599 _step‚460 2loss/SRqqt2/assignment_loss‚0.6286483715572048 2loss/FBt2/assignment_loss‚0.036580490762410725 1loss/SRqqt2/detection_loss‚0.7020666066183103 epoch‚13 /loss/FRt2/assignment_loss‚1.106175001852498 /loss/FRt2/detection_loss‚1.4742937320701028 /loss/FBt2/detection_loss‚1.2149479350902967 _runtime‚3875.919965656Ì_ópuª‚(:ô Ò†óÒÑ¡”¥ network.recv‚ 1.033282e+06 disk.dm-1.in‚0 disk.nvme0n1.in‚ 1852.4453125& disk.nvme1n1.out‚16772.16845703125 disk.md0.in‚0 disk.dm-0.in‚ 1982.2109375' proc.memory.availableMB‚ 57.83203125* proc.memory.percent‚17.811059951782227% disk./.usageGB‚1172.3729209899902" disk.dm-1.out‚2002.41552734375" disk.nvme0c0n1.in‚ 1852.4453125 disk.nvme1c1n1.in‚ 129.765625" proc.memory.rssMB‚ 2918.1640625 cpu‚25.900069666289205 disk.rbd0.in‚0 disk.rbd0.out‚0 disk.nvme1n1.in‚ 129.765625$ memory_percent‚99.64702129364014 proc.cpu.threads‚17 network.sent‚ 8.333294e+06) disk./.usagePercent‚76.36364898159846( disk.nvme0c0n1.out‚40008.04931640625& disk.nvme0n1.out‚40008.04931640625( disk.nvme1c1n1.out‚16772.16845703125 disk.md0.out‚ 94.51953125# disk.dm-0.out‚54574.17919921875]*sø«‚(:’ Ò†óÒÜõÍ¥ gpu.0.gpu‚60.0 gpu.0.memory‚9, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚48.0 gpu.0.powerWatts‚153.122' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚51.04066666666667 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ÝÅžÍ2¬- _wandbruntime‚3889  _runtime‚3889O;Ísà=­Ú{ :"Purity/*FRt0SRqqt*FBt/event_purity‚0.20655956864356995 8 Purity/*FRt0SRqqt*FBt/FRt_purity‚0.14925938844680786 7 Purity/1FRt*SRqqt*FBt/FBt_purity‚0.9968254566192627 9"Purity/1FRt1SRqqt*FBt/SRqqt_purity‚0.7402310967445374 9"Purity/2FRt1SRqqt1FBt/SRqqt_purity‚0.7777777910232544 8 Purity/*FRt0SRqqt0FBt/FRt_purity‚0.13418027758598328 9"Purity/*FRt*SRqqt*FBt/SRqqt_purity‚0.6655823588371277 A&Purity/2FRt0SRqqt2FBt/event_proportion‚0.00018601190822664648 & Purity/2FRt0SRqqt1FBt/FBt_purity‚1 :"Purity/*FRt1SRqqt2FBt/SRqqt_purity‚0.12962964177131653 9"Purity/*FRt2SRqqt0FBt/event_purity‚0.1571653038263321 7 Purity/0FRt0SRqqt1FBt/FBt_purity‚0.9974045753479004 9"Purity/2FRt0SRqqt2FBt/event_purity‚0.5714285969734192 & Purity/*FRt2SRqqt1FBt/FRt_purity‚0 7 Purity/0FRt*SRqqt2FBt/FBt_purity‚0.9832104444503784 9"Purity/0FRt1SRqqt*FBt/event_purity‚0.6151406168937683 >&Purity/1FRt0SRqqt1FBt/event_proportion‚0.00969587080180645 ("Purity/1FRt2SRqqt1FBt/event_purity‚0 particle/accuracy_2_of_4‚1 ?&Purity/0FRt1SRqqt1FBt/event_proportion‚0.011555989272892475 8 Purity/*FRt*SRqqt0FBt/FRt_purity‚0.12396940588951111 8 Purity/*FRt1SRqqt*FBt/FRt_purity‚0.12446548044681549 7 Purity/1FRt*SRqqt1FBt/FRt_purity‚0.5560402274131775 & Purity/1FRt1SRqqt1FBt/FBt_purity‚1 & Purity/2FRt1SRqqt1FBt/FBt_purity‚1 _runtime‚3889.519467498 8 Purity/*FRt1SRqqt0FBt/FRt_purity‚0.10655311495065689 8 Purity/*FRt*SRqqt*FBt/FRt_purity‚0.13894303143024445 9"Purity/0FRt*SRqqt*FBt/SRqqt_purity‚0.6123356819152832 9"Purity/0FRt0SRqqt1FBt/event_purity‚0.9974045753479004 & Purity/2FRt*SRqqt2FBt/FBt_purity‚1 particle/accuracy_3_of_5‚1 ("Purity/0FRt0SRqqt0FBt/event_purity‚1 , Purity/0FRt2SRqqt*FBt/FBt_purity‚0.96875 ("Purity/2FRt1SRqqt0FBt/event_purity‚0 9"Purity/2FRt2SRqqt*FBt/SRqqt_purity‚0.7622491121292114 ?&Purity/2FRt2SRqqt0FBt/event_proportion‚0.006417410913854837 9"Purity/0FRt0SRqqt2FBt/event_purity‚0.9753291010856628 9"Purity/1FRt*SRqqt*FBt/SRqqt_purity‚0.7267536520957947 9"Purity/0FRt2SRqqt*FBt/event_purity‚0.3460543155670166 =&Purity/1FRt*SRqqt*FBt/event_proportion‚0.6297432780265808 :"Purity/1FRt*SRqqt0FBt/event_purity‚0.10252854973077774 (particle/accuracy‚0.8494349122047424 9"Purity/0FRt*SRqqt2FBt/SRqqt_purity‚0.1538461595773697 jet/accuracy_5_of_5‚0 7 Purity/1FRt0SRqqt2FBt/FBt_purity‚0.9970238208770752 particle/accuracy_4_of_5‚1 9"Purity/*FRt2SRqqt1FBt/SRqqt_purity‚0.3680555820465088 _step‚461 <&Purity/*FRt*SRqqt0FBt/event_proportion‚0.881045401096344 9"Purity/1FRt*SRqqt2FBt/SRqqt_purity‚0.1111111119389534 :"Purity/1FRt0SRqqt0FBt/event_purity‚0.13449737429618835 ("Purity/1FRt2SRqqt*FBt/event_purity‚0 8&Purity/1FRt2SRqqt0FBt/event_proportion‚ 0.02294921875 9"Purity/0FRt1SRqqt2FBt/SRqqt_purity‚0.1538461595773697 =&Purity/*FRt*SRqqt1FBt/event_proportion‚0.0617094486951828 9"Purity/*FRt2SRqqt*FBt/SRqqt_purity‚0.6459209322929382 <&Purity/2FRt*SRqqt*FBt/event_proportion‚0.040062315762043 ,&Purity/2FRt2SRqqt1FBt/event_proportion‚0 7 Purity/0FRt1SRqqt2FBt/FBt_purity‚0.9230769276618958 7 Purity/*FRt0SRqqt1FBt/FBt_purity‚0.9981421232223511 9"Purity/0FRt*SRqqt0FBt/event_purity‚0.5964857935905457 8"Purity/0FRt0SRqqt*FBt/event_purity‚0.993191123008728 :"Purity/1FRt1SRqqt*FBt/event_purity‚0.03159944340586662 @&Purity/1FRt1SRqqt1FBt/event_proportion‚0.0064639137126505375 8 Purity/2FRt0SRqqt0FBt/FRt_purity‚0.13133451342582703 )jet/accuracy_2_of_4‚0.719189465045929 9"Purity/*FRt*SRqqt*FBt/event_purity‚0.2670508325099945 ("Purity/*FRt2SRqqt1FBt/event_purity‚0 7 Purity/0FRt1SRqqt*FBt/FBt_purity‚0.9913028478622437 >&Purity/1FRt*SRqqt1FBt/event_proportion‚0.01643880270421505 7 Purity/1FRt0SRqqt1FBt/FRt_purity‚0.6194849014282227 @&Purity/2FRt*SRqqt2FBt/event_proportion‚0.0002092633949359879 7 Purity/2FRt0SRqqt*FBt/FRt_purity‚0.1527484506368637 9"Purity/2FRt0SRqqt1FBt/event_purity‚0.2604166865348816 & Purity/1FRt2SRqqt*FBt/FBt_purity‚1 +jet/accuracy_1_of_1‚0.28991830348968506 particle/accuracy_1_of_5‚1 trainer/global_step‚5599 ,&Purity/*FRt2SRqqt2FBt/event_proportion‚0 9"Purity/2FRt2SRqqt0FBt/SRqqt_purity‚0.7622491121292114 7 Purity/*FRt1SRqqt1FBt/FBt_purity‚0.9987789392471313 9"Purity/0FRt2SRqqt0FBt/SRqqt_purity‚0.5925097465515137 8 Purity/1FRt1SRqqt*FBt/FRt_purity‚0.11657353490591049 =&Purity/2FRt0SRqqt*FBt/event_proportion‚0.0208565853536129 8 Purity/2FRt1SRqqt*FBt/FRt_purity‚0.17036685347557068 *jet/accuracy_1_of_4‚0.9471265077590942 8 Purity/2FRt1SRqqt0FBt/FRt_purity‚0.15041053295135498 7 Purity/*FRt*SRqqt1FBt/FBt_purity‚0.9980267882347107 8&Purity/*FRt2SRqqt0FBt/event_proportion‚ 0.05322265625 9"Purity/0FRt*SRqqt*FBt/event_purity‚0.6431729197502136 =&Purity/0FRt0SRqqt*FBt/event_proportion‚0.0390159972012043 >&Purity/0FRt2SRqqt0FBt/event_proportion‚0.02385602705180645 7 Purity/1FRt0SRqqt*FBt/FBt_purity‚0.9988385438919067 8 Purity/1FRt1SRqqt0FBt/FRt_purity‚0.09919049590826035 =&Purity/*FRt1SRqqt*FBt/event_proportion‚0.3866024911403656 & Purity/1FRt2SRqqt1FBt/FRt_purity‚0 =&Purity/2FRt*SRqqt0FBt/event_proportion‚0.0382486991584301 *"Purity/2FRt*SRqqt2FBt/event_purity‚0.5 =&Purity/*FRt0SRqqt0FBt/event_proportion‚0.4607747495174408 9"Purity/0FRt1SRqqt*FBt/SRqqt_purity‚0.6186209917068481 @&Purity/1FRt0SRqqt2FBt/event_proportion‚0.0024879092816263437 7 Purity/1FRt0SRqqt2FBt/FRt_purity‚0.8872448801994324 ;"Purity/2FRt0SRqqt*FBt/event_purity‚0.027902359142899513 )loss/val_total_loss‚9.644850333992977 9"Purity/0FRt2SRqqt*FBt/SRqqt_purity‚0.5868300199508667 :"Purity/1FRt2SRqqt1FBt/SRqqt_purity‚0.32499998807907104 particle/accuracy_1_of_3‚1 9"Purity/*FRt*SRqqt0FBt/event_purity‚0.2285788208246231 9"Purity/*FRt1SRqqt0FBt/SRqqt_purity‚0.6771634817123413 ,&Purity/0FRt2SRqqt2FBt/event_proportion‚0 9"Purity/1FRt1SRqqt2FBt/SRqqt_purity‚0.1111111119389534 >&Purity/1FRt2SRqqt*FBt/event_proportion‚0.02322823740541935 & Purity/2FRt*SRqqt*FBt/FBt_purity‚1 =&Purity/0FRt*SRqqt1FBt/event_proportion‚0.0436662957072258 @&Purity/2FRt1SRqqt2FBt/event_proportion‚2.325148852833081e-05 ("Purity/2FRt2SRqqt*FBt/event_purity‚0 ?&Purity/*FRt*SRqqt2FBt/event_proportion‚0.010997953824698925 =&Purity/*FRt0SRqqt1FBt/event_proportion‚0.0418526791036129 7 Purity/0FRt*SRqqt*FBt/FBt_purity‚0.9932601451873779 9"Purity/0FRt1SRqqt0FBt/event_purity‚0.6246905326843262 9"Purity/1FRt*SRqqt*FBt/event_purity‚0.1135038435459137 <"Purity/2FRt*SRqqt0FBt/event_purity‚0.0077096084132790565 ) Purity/2FRt*SRqqt2FBt/FRt_purity‚0.75 7 Purity/1FRt*SRqqt2FBt/FBt_purity‚0.9894558191299438 9"Purity/1FRt2SRqqt*FBt/SRqqt_purity‚0.6821696162223816 :"Purity/*FRt2SRqqt*FBt/event_purity‚0.15426039695739746 7 Purity/2FRt*SRqqt1FBt/FRt_purity‚0.5133333206176758 /particle/accuracy_3_of_3‚0.9994933605194092 jet/accuracy_1_of_5‚1 ?&Purity/2FRt1SRqqt0FBt/event_proportion‚0.011928013525903225 =&Purity/*FRt1SRqqt1FBt/event_proportion‚0.0188569575548172 ("Purity/1FRt2SRqqt0FBt/event_purity‚0 'particle/f_score‚0.6427584290504456 ("Purity/2FRt1SRqqt2FBt/event_purity‚0 =&Purity/2FRt1SRqqt*FBt/event_proportion‚0.0127883180975914 ("Purity/2FRt1SRqqt1FBt/event_purity‚0 =&Purity/*FRt0SRqqt*FBt/event_proportion‚0.5129278302192688 9"Purity/*FRt0SRqqt0FBt/event_purity‚0.1293083280324936 9"Purity/1FRt1SRqqt0FBt/SRqqt_purity‚0.7458270192146301 ,jet/accuracy_4_of_4‚0.013899499550461769 particle/accuracy_2_of_2‚1 6validation_average_jet_accuracy‚0.3381672501564026 & Purity/2FRt0SRqqt2FBt/FBt_purity‚1 9"Purity/*FRt1SRqqt1FBt/SRqqt_purity‚0.5750026702880859 9"Purity/*FRt2SRqqt0FBt/SRqqt_purity‚0.6510157585144043 +jet/accuracy_3_of_3‚0.06437167525291443 particle/accuracy_1_of_2‚1 9"Purity/1FRt*SRqqt1FBt/SRqqt_purity‚0.6158331632614136 ("Purity/1FRt1SRqqt2FBt/event_purity‚0 particle/accuracy_5_of_5‚0 9"Purity/*FRt0SRqqt2FBt/event_purity‚0.9197235107421875 ?&Purity/0FRt*SRqqt2FBt/event_proportion‚0.008068266324698925 @&Purity/1FRt*SRqqt2FBt/event_proportion‚0.0027204242069274187 :"Purity/1FRt1SRqqt1FBt/event_purity‚0.22856537997722626 & Purity/2FRt*SRqqt1FBt/FBt_purity‚1 =&Purity/0FRt1SRqqt0FBt/event_proportion‚0.2083565890789032 ("Purity/0FRt2SRqqt1FBt/event_purity‚0 9"Purity/1FRt0SRqqt2FBt/event_purity‚0.7789115905761719 7 Purity/1FRt1SRqqt2FBt/FBt_purity‚0.8888888955116272 >&Purity/2FRt0SRqqt0FBt/event_proportion‚0.01990327425301075 ("Purity/2FRt1SRqqt2FBt/SRqqt_purity‚0 & Purity/2FRt1SRqqt2FBt/FBt_purity‚1 7 Purity/0FRt*SRqqt1FBt/FBt_purity‚0.9971537590026855 A&Purity/1FRt2SRqqt1FBt/event_proportion‚0.00027901786961592734 ?&Purity/2FRt2SRqqt*FBt/event_proportion‚0.006417410913854837 /particle/accuracy_3_of_4‚0.9942177534103394 9"Purity/*FRt*SRqqt1FBt/SRqqt_purity‚0.5550576448440552 6 Purity/0FRt0SRqqt2FBt/FBt_purity‚0.987664520740509 7 Purity/*FRt*SRqqt*FBt/FBt_purity‚0.9944456815719604 @&Purity/0FRt2SRqqt1FBt/event_proportion‚0.0007207961170934141 & Purity/1FRt0SRqqt1FBt/FBt_purity‚1 % _timestamp‚1.7844642151305945e+09 7 Purity/*FRt0SRqqt1FBt/FRt_purity‚0.6058465242385864 7 Purity/*FRt1SRqqt*FBt/FBt_purity‚0.9920156598091125 9"Purity/0FRt*SRqqt1FBt/SRqqt_purity‚0.5136484503746033 9"Purity/2FRt*SRqqt*FBt/SRqqt_purity‚0.7550957798957825 7 Purity/*FRt*SRqqt2FBt/FRt_purity‚0.8795163035392761 9"Purity/0FRt1SRqqt1FBt/SRqqt_purity‚0.5291898846626282 @&Purity/0FRt1SRqqt2FBt/event_proportion‚0.0004417782765813172 =&Purity/1FRt1SRqqt0FBt/event_proportion‚0.1467633992433548 7 Purity/*FRt1SRqqt2FBt/FRt_purity‚0.8500000238418579 7 Purity/*FRt*SRqqt1FBt/FRt_purity‚0.5441954731941223 =&Purity/*FRt0SRqqt2FBt/event_proportion‚0.0103004090487957 6 Purity/0FRt1SRqqt1FBt/FBt_purity‚0.998015820980072 8 Purity/1FRt*SRqqt0FBt/FRt_purity‚0.12353473901748657 :"Purity/2FRt*SRqqt1FBt/event_purity‚0.12166666984558105 @&Purity/2FRt0SRqqt1FBt/event_proportion‚0.0007672990905120969 7 Purity/*FRt0SRqqt*FBt/FBt_purity‚0.9955178499221802 7 Purity/*FRt1SRqqt1FBt/FRt_purity‚0.4754045009613037 8 Purity/1FRt*SRqqt*FBt/FRt_purity‚0.13811200857162476 ,&Purity/1FRt2SRqqt2FBt/event_proportion‚0 9"Purity/2FRt1SRqqt0FBt/SRqqt_purity‚0.7523024678230286 +jet/accuracy_3_of_4‚0.09065638482570648 particle/accuracy_1_of_1‚1 ("Purity/2FRt*SRqqt2FBt/SRqqt_purity‚0 9"Purity/*FRt1SRqqt*FBt/SRqqt_purity‚0.6711901426315308 8 Purity/*FRt2SRqqt*FBt/FRt_purity‚0.06956833600997925 @&Purity/2FRt1SRqqt1FBt/event_proportion‚0.0008370535797439516 +jet/accuracy_2_of_2‚0.12732656300067902 *jet/accuracy_1_of_3‚0.9126162528991699 7 Purity/*FRt0SRqqt2FBt/FRt_purity‚0.8822373747825623 =&Purity/*FRt1SRqqt0FBt/event_proportion‚0.3670479953289032 8 Purity/*FRt2SRqqt0FBt/FRt_purity‚0.07013906538486481 9"Purity/1FRt*SRqqt0FBt/SRqqt_purity‚0.7317684888839722 :"Purity/1FRt0SRqqt*FBt/event_purity‚0.14710505306720734 & Purity/1FRt2SRqqt1FBt/FBt_purity‚1 9"Purity/2FRt1SRqqt*FBt/SRqqt_purity‚0.7531779408454895 jet/accuracy_2_of_5‚1 9"Purity/0FRt*SRqqt2FBt/event_purity‚0.9183914661407471 :"Purity/0FRt2SRqqt0FBt/event_purity‚0.35620003938674927 9"Purity/2FRt*SRqqt1FBt/SRqqt_purity‚0.7777777910232544 particle/accuracy_2_of_3‚1 7 Purity/*FRt0SRqqt2FBt/FBt_purity‚0.9900391101837158 @&Purity/*FRt1SRqqt2FBt/event_proportion‚0.0006975446594879031 =&Purity/0FRt1SRqqt*FBt/event_proportion‚0.2203543484210968 9"Purity/0FRt*SRqqt1FBt/event_purity‚0.8388054966926575 =&Purity/0FRt0SRqqt0FBt/event_proportion‚0.0462472103536129 =&Purity/0FRt0SRqqt1FBt/event_proportion‚0.0313895083963871 <&Purity/*FRt2SRqqt*FBt/event_proportion‚0.054222472012043 @&Purity/*FRt2SRqqt1FBt/event_proportion‚0.0009998140158131719 9"Purity/0FRt*SRqqt0FBt/SRqqt_purity‚0.6182425022125244 , Purity/0FRt2SRqqt1FBt/FBt_purity‚0.96875 =&Purity/1FRt*SRqqt0FBt/event_proportion‚0.6105840802192688 @&Purity/2FRt*SRqqt1FBt/event_proportion‚0.0016043527284637094 ("Purity/0FRt1SRqqt2FBt/event_purity‚0 7 Purity/2FRt*SRqqt*FBt/FRt_purity‚0.1454634666442871 ;"Purity/2FRt0SRqqt0FBt/event_purity‚0.014101014472544193 7 Purity/*FRt*SRqqt2FBt/FBt_purity‚0.9843401312828064 9"Purity/1FRt*SRqqt2FBt/event_purity‚0.7132652997970581 =&Purity/1FRt1SRqqt*FBt/event_proportion‚0.1534598171710968 ("Purity/2FRt2SRqqt0FBt/event_purity‚0 particle/accuracy_1_of_4‚1 /particle/accuracy_4_of_4‚0.9497403502464294 7 Purity/*FRt1SRqqt2FBt/FBt_purity‚0.9074073433876038 9"Purity/0FRt1SRqqt1FBt/event_purity‚0.4740719497203827 =&Purity/1FRt0SRqqt0FBt/event_proportion‚0.4408714771270752 8 Purity/2FRt*SRqqt0FBt/FRt_purity‚0.12737533450126648 ,&Purity/2FRt2SRqqt2FBt/event_proportion‚0 +particle/specificity‚0.9092310070991516 epoch‚13 =&Purity/*FRt*SRqqt*FBt/event_proportion‚0.9537528157234192 ."Purity/0FRt2SRqqt1FBt/SRqqt_purity‚0.40625 8 Purity/2FRt2SRqqt0FBt/FRt_purity‚0.07285372167825699 particle/accuracy_2_of_5‚1 9"Purity/*FRt*SRqqt2FBt/event_purity‚0.8599764704704285 =&Purity/0FRt*SRqqt0FBt/event_proportion‚0.2322126179933548 7 Purity/0FRt0SRqqt*FBt/FBt_purity‚0.9943971037864685 9"Purity/1FRt1SRqqt1FBt/SRqqt_purity‚0.6342597007751465 7 Purity/1FRt1SRqqt2FBt/FRt_purity‚0.8888888955116272 8 Purity/2FRt2SRqqt*FBt/FRt_purity‚0.07285372167825699 )jet/accuracy_2_of_3‚0.490396648645401 ( Purity/2FRt1SRqqt2FBt/FRt_purity‚0.5 9"Purity/*FRt*SRqqt1FBt/event_purity‚0.7090440392494202 8 Purity/1FRt0SRqqt*FBt/FRt_purity‚0.14899876713752747 ;"Purity/1FRt1SRqqt0FBt/event_purity‚0.022641528397798538 @&Purity/1FRt1SRqqt2FBt/event_proportion‚0.0002325148816453293 7 Purity/2FRt0SRqqt1FBt/FRt_purity‚0.5651041269302368 8 Purity/1FRt0SRqqt0FBt/FRt_purity‚0.13449737429618835 9"Purity/1FRt0SRqqt1FBt/event_purity‚0.5580739974975586 7 Purity/1FRt1SRqqt*FBt/FBt_purity‚0.9934239983558655 8"Purity/1FRt2SRqqt0FBt/SRqqt_purity‚0.686802864074707 :"Purity/2FRt*SRqqt*FBt/event_purity‚0.01484821829944849 & Purity/2FRt1SRqqt*FBt/FBt_purity‚1 jet/accuracy_3_of_5‚1 9"Purity/*FRt*SRqqt0FBt/SRqqt_purity‚0.6712260246276855 9"Purity/*FRt1SRqqt*FBt/event_purity‚0.3633514940738678 9"Purity/*FRt1SRqqt0FBt/event_purity‚0.3638712465763092 :"Purity/*FRt1SRqqt1FBt/event_purity‚0.36844807863235474 7 Purity/*FRt2SRqqt1FBt/FBt_purity‚0.9814814329147339 8 Purity/1FRt2SRqqt0FBt/FRt_purity‚0.06808863580226898 +particle/sensitivity‚0.6307960748672485 7 Purity/2FRt0SRqqt2FBt/FRt_purity‚0.7857142686843872 9"Purity/*FRt0SRqqt1FBt/event_purity‚0.8811167478561401 8 Purity/1FRt2SRqqt*FBt/FRt_purity‚0.06714209169149399 ("Purity/2FRt1SRqqt*FBt/event_purity‚0 9"Purity/0FRt1SRqqt0FBt/SRqqt_purity‚0.6246905326843262 9"Purity/2FRt*SRqqt0FBt/SRqqt_purity‚0.7547366619110107 *jet/accuracy_1_of_2‚0.7789285778999329 & Purity/2FRt0SRqqt*FBt/FBt_purity‚1 7 Purity/2FRt1SRqqt1FBt/FRt_purity‚0.4699074327945709 >&Purity/0FRt2SRqqt*FBt/event_proportion‚0.02457682229578495 ("Purity/*FRt1SRqqt2FBt/event_purity‚0 :"Purity/1FRt*SRqqt1FBt/event_purity‚0.42158639430999756 & Purity/1FRt*SRqqt1FBt/FBt_purity‚1 7 Purity/1FRt1SRqqt1FBt/FRt_purity‚0.4754408001899719 jet/accuracy_4_of_5‚0 7 Purity/1FRt*SRqqt2FBt/FRt_purity‚0.8910998106002808 7 Purity/*FRt2SRqqt*FBt/FBt_purity‚0.9814814329147339 =&Purity/0FRt*SRqqt*FBt/event_proportion‚0.2839471697807312 ?&Purity/0FRt0SRqqt2FBt/event_proportion‚0.007626487873494625 =&Purity/1FRt0SRqqt*FBt/event_proportion‚0.4530552327632904 :"Purity/*FRt*SRqqt2FBt/SRqqt_purity‚0.12962964177131653ãu‹éÛ.®ß{ =&Purity/*FRt0SRqqt2FBt/event_proportion‚0.0103004090487957 6 Purity/0FRt1SRqqt1FBt/FBt_purity‚0.998015820980072 8 Purity/1FRt*SRqqt0FBt/FRt_purity‚0.12353473901748657 :"Purity/2FRt*SRqqt1FBt/event_purity‚0.12166666984558105 @&Purity/2FRt0SRqqt1FBt/event_proportion‚0.0007672990905120969 7 Purity/*FRt0SRqqt*FBt/FBt_purity‚0.9955178499221802 7 Purity/*FRt1SRqqt1FBt/FRt_purity‚0.4754045009613037 8 Purity/1FRt*SRqqt*FBt/FRt_purity‚0.13811200857162476 ,&Purity/1FRt2SRqqt2FBt/event_proportion‚0 9"Purity/2FRt1SRqqt0FBt/SRqqt_purity‚0.7523024678230286 +jet/accuracy_3_of_4‚0.09065638482570648 particle/accuracy_1_of_1‚1 ("Purity/2FRt*SRqqt2FBt/SRqqt_purity‚0 9"Purity/*FRt1SRqqt*FBt/SRqqt_purity‚0.6711901426315308 8 Purity/*FRt2SRqqt*FBt/FRt_purity‚0.06956833600997925 @&Purity/2FRt1SRqqt1FBt/event_proportion‚0.0008370535797439516 +jet/accuracy_2_of_2‚0.12732656300067902 *jet/accuracy_1_of_3‚0.9126162528991699 7 Purity/*FRt0SRqqt2FBt/FRt_purity‚0.8822373747825623 =&Purity/*FRt1SRqqt0FBt/event_proportion‚0.3670479953289032 8 Purity/*FRt2SRqqt0FBt/FRt_purity‚0.07013906538486481 9"Purity/1FRt*SRqqt0FBt/SRqqt_purity‚0.7317684888839722 :"Purity/1FRt0SRqqt*FBt/event_purity‚0.14710505306720734 & Purity/1FRt2SRqqt1FBt/FBt_purity‚1 9"Purity/2FRt1SRqqt*FBt/SRqqt_purity‚0.7531779408454895 jet/accuracy_2_of_5‚1 9"Purity/0FRt*SRqqt2FBt/event_purity‚0.9183914661407471 :"Purity/0FRt2SRqqt0FBt/event_purity‚0.35620003938674927 9"Purity/2FRt*SRqqt1FBt/SRqqt_purity‚0.7777777910232544 particle/accuracy_2_of_3‚1 7 Purity/*FRt0SRqqt2FBt/FBt_purity‚0.9900391101837158 @&Purity/*FRt1SRqqt2FBt/event_proportion‚0.0006975446594879031 =&Purity/0FRt1SRqqt*FBt/event_proportion‚0.2203543484210968 9"Purity/0FRt*SRqqt1FBt/event_purity‚0.8388054966926575 =&Purity/0FRt0SRqqt0FBt/event_proportion‚0.0462472103536129 =&Purity/0FRt0SRqqt1FBt/event_proportion‚0.0313895083963871 <&Purity/*FRt2SRqqt*FBt/event_proportion‚0.054222472012043 @&Purity/*FRt2SRqqt1FBt/event_proportion‚0.0009998140158131719 9"Purity/0FRt*SRqqt0FBt/SRqqt_purity‚0.6182425022125244 , Purity/0FRt2SRqqt1FBt/FBt_purity‚0.96875 =&Purity/1FRt*SRqqt0FBt/event_proportion‚0.6105840802192688 @&Purity/2FRt*SRqqt1FBt/event_proportion‚0.0016043527284637094 ("Purity/0FRt1SRqqt2FBt/event_purity‚0 7 Purity/2FRt*SRqqt*FBt/FRt_purity‚0.1454634666442871 ;"Purity/2FRt0SRqqt0FBt/event_purity‚0.014101014472544193 7 Purity/*FRt*SRqqt2FBt/FBt_purity‚0.9843401312828064 9"Purity/1FRt*SRqqt2FBt/event_purity‚0.7132652997970581 =&Purity/1FRt1SRqqt*FBt/event_proportion‚0.1534598171710968 ("Purity/2FRt2SRqqt0FBt/event_purity‚0 particle/accuracy_1_of_4‚1 /particle/accuracy_4_of_4‚0.9497403502464294 7 Purity/*FRt1SRqqt2FBt/FBt_purity‚0.9074073433876038 9"Purity/0FRt1SRqqt1FBt/event_purity‚0.4740719497203827 =&Purity/1FRt0SRqqt0FBt/event_proportion‚0.4408714771270752 8 Purity/2FRt*SRqqt0FBt/FRt_purity‚0.12737533450126648 ,&Purity/2FRt2SRqqt2FBt/event_proportion‚0 +particle/specificity‚0.9092310070991516 epoch‚13 =&Purity/*FRt*SRqqt*FBt/event_proportion‚0.9537528157234192 ."Purity/0FRt2SRqqt1FBt/SRqqt_purity‚0.40625 8 Purity/2FRt2SRqqt0FBt/FRt_purity‚0.07285372167825699 particle/accuracy_2_of_5‚1 9"Purity/*FRt*SRqqt2FBt/event_purity‚0.8599764704704285 =&Purity/0FRt*SRqqt0FBt/event_proportion‚0.2322126179933548 7 Purity/0FRt0SRqqt*FBt/FBt_purity‚0.9943971037864685 9"Purity/1FRt1SRqqt1FBt/SRqqt_purity‚0.6342597007751465 7 Purity/1FRt1SRqqt2FBt/FRt_purity‚0.8888888955116272 8 Purity/2FRt2SRqqt*FBt/FRt_purity‚0.07285372167825699 )jet/accuracy_2_of_3‚0.490396648645401 ( Purity/2FRt1SRqqt2FBt/FRt_purity‚0.5 9"Purity/*FRt*SRqqt1FBt/event_purity‚0.7090440392494202 8 Purity/1FRt0SRqqt*FBt/FRt_purity‚0.14899876713752747 ;"Purity/1FRt1SRqqt0FBt/event_purity‚0.022641528397798538 @&Purity/1FRt1SRqqt2FBt/event_proportion‚0.0002325148816453293 7 Purity/2FRt0SRqqt1FBt/FRt_purity‚0.5651041269302368 8 Purity/1FRt0SRqqt0FBt/FRt_purity‚0.13449737429618835 9"Purity/1FRt0SRqqt1FBt/event_purity‚0.5580739974975586 7 Purity/1FRt1SRqqt*FBt/FBt_purity‚0.9934239983558655 8"Purity/1FRt2SRqqt0FBt/SRqqt_purity‚0.686802864074707 :"Purity/2FRt*SRqqt*FBt/event_purity‚0.01484821829944849 & Purity/2FRt1SRqqt*FBt/FBt_purity‚1 jet/accuracy_3_of_5‚1 9"Purity/*FRt*SRqqt0FBt/SRqqt_purity‚0.6712260246276855 9"Purity/*FRt1SRqqt*FBt/event_purity‚0.3633514940738678 9"Purity/*FRt1SRqqt0FBt/event_purity‚0.3638712465763092 :"Purity/*FRt1SRqqt1FBt/event_purity‚0.36844807863235474 7 Purity/*FRt2SRqqt1FBt/FBt_purity‚0.9814814329147339 8 Purity/1FRt2SRqqt0FBt/FRt_purity‚0.06808863580226898 +particle/sensitivity‚0.6307960748672485 7 Purity/2FRt0SRqqt2FBt/FRt_purity‚0.7857142686843872 9"Purity/*FRt0SRqqt1FBt/event_purity‚0.8811167478561401 8 Purity/1FRt2SRqqt*FBt/FRt_purity‚0.06714209169149399 ("Purity/2FRt1SRqqt*FBt/event_purity‚0 9"Purity/0FRt1SRqqt0FBt/SRqqt_purity‚0.6246905326843262 9"Purity/2FRt*SRqqt0FBt/SRqqt_purity‚0.7547366619110107 *jet/accuracy_1_of_2‚0.7789285778999329 & Purity/2FRt0SRqqt*FBt/FBt_purity‚1 7 Purity/2FRt1SRqqt1FBt/FRt_purity‚0.4699074327945709 >&Purity/0FRt2SRqqt*FBt/event_proportion‚0.02457682229578495 ("Purity/*FRt1SRqqt2FBt/event_purity‚0 :"Purity/1FRt*SRqqt1FBt/event_purity‚0.42158639430999756 & Purity/1FRt*SRqqt1FBt/FBt_purity‚1 7 Purity/1FRt1SRqqt1FBt/FRt_purity‚0.4754408001899719 jet/accuracy_4_of_5‚0 7 Purity/1FRt*SRqqt2FBt/FRt_purity‚0.8910998106002808 7 Purity/*FRt2SRqqt*FBt/FBt_purity‚0.9814814329147339 =&Purity/0FRt*SRqqt*FBt/event_proportion‚0.2839471697807312 ?&Purity/0FRt0SRqqt2FBt/event_proportion‚0.007626487873494625 =&Purity/1FRt0SRqqt*FBt/event_proportion‚0.4530552327632904 :"Purity/*FRt*SRqqt2FBt/SRqqt_purity‚0.12962964177131653 :"Purity/*FRt0SRqqt*FBt/event_purity‚0.20655956864356995 8 Purity/*FRt0SRqqt*FBt/FRt_purity‚0.14925938844680786 7 Purity/1FRt*SRqqt*FBt/FBt_purity‚0.9968254566192627 9"Purity/1FRt1SRqqt*FBt/SRqqt_purity‚0.7402310967445374 9"Purity/2FRt1SRqqt1FBt/SRqqt_purity‚0.7777777910232544 8 Purity/*FRt0SRqqt0FBt/FRt_purity‚0.13418027758598328 9"Purity/*FRt*SRqqt*FBt/SRqqt_purity‚0.6655823588371277 A&Purity/2FRt0SRqqt2FBt/event_proportion‚0.00018601190822664648 & Purity/2FRt0SRqqt1FBt/FBt_purity‚1 :"Purity/*FRt1SRqqt2FBt/SRqqt_purity‚0.12962964177131653 9"Purity/*FRt2SRqqt0FBt/event_purity‚0.1571653038263321 7 Purity/0FRt0SRqqt1FBt/FBt_purity‚0.9974045753479004 9"Purity/2FRt0SRqqt2FBt/event_purity‚0.5714285969734192 & Purity/*FRt2SRqqt1FBt/FRt_purity‚0 7 Purity/0FRt*SRqqt2FBt/FBt_purity‚0.9832104444503784 9"Purity/0FRt1SRqqt*FBt/event_purity‚0.6151406168937683 >&Purity/1FRt0SRqqt1FBt/event_proportion‚0.00969587080180645 ("Purity/1FRt2SRqqt1FBt/event_purity‚0 particle/accuracy_2_of_4‚1 ?&Purity/0FRt1SRqqt1FBt/event_proportion‚0.011555989272892475 8 Purity/*FRt*SRqqt0FBt/FRt_purity‚0.12396940588951111 8 Purity/*FRt1SRqqt*FBt/FRt_purity‚0.12446548044681549 7 Purity/1FRt*SRqqt1FBt/FRt_purity‚0.5560402274131775 & Purity/1FRt1SRqqt1FBt/FBt_purity‚1 & Purity/2FRt1SRqqt1FBt/FBt_purity‚1 _runtime‚3889.519467498 8 Purity/*FRt1SRqqt0FBt/FRt_purity‚0.10655311495065689 8 Purity/*FRt*SRqqt*FBt/FRt_purity‚0.13894303143024445 9"Purity/0FRt*SRqqt*FBt/SRqqt_purity‚0.6123356819152832 9"Purity/0FRt0SRqqt1FBt/event_purity‚0.9974045753479004 & Purity/2FRt*SRqqt2FBt/FBt_purity‚1 particle/accuracy_3_of_5‚1 ("Purity/0FRt0SRqqt0FBt/event_purity‚1 , Purity/0FRt2SRqqt*FBt/FBt_purity‚0.96875 ("Purity/2FRt1SRqqt0FBt/event_purity‚0 9"Purity/2FRt2SRqqt*FBt/SRqqt_purity‚0.7622491121292114 ?&Purity/2FRt2SRqqt0FBt/event_proportion‚0.006417410913854837 9"Purity/0FRt0SRqqt2FBt/event_purity‚0.9753291010856628 9"Purity/1FRt*SRqqt*FBt/SRqqt_purity‚0.7267536520957947 9"Purity/0FRt2SRqqt*FBt/event_purity‚0.3460543155670166 =&Purity/1FRt*SRqqt*FBt/event_proportion‚0.6297432780265808 :"Purity/1FRt*SRqqt0FBt/event_purity‚0.10252854973077774 (particle/accuracy‚0.8494349122047424 9"Purity/0FRt*SRqqt2FBt/SRqqt_purity‚0.1538461595773697 jet/accuracy_5_of_5‚0 7 Purity/1FRt0SRqqt2FBt/FBt_purity‚0.9970238208770752 particle/accuracy_4_of_5‚1 9"Purity/*FRt2SRqqt1FBt/SRqqt_purity‚0.3680555820465088 _step‚461 <&Purity/*FRt*SRqqt0FBt/event_proportion‚0.881045401096344 9"Purity/1FRt*SRqqt2FBt/SRqqt_purity‚0.1111111119389534 :"Purity/1FRt0SRqqt0FBt/event_purity‚0.13449737429618835 ("Purity/1FRt2SRqqt*FBt/event_purity‚0 8&Purity/1FRt2SRqqt0FBt/event_proportion‚ 0.02294921875 9"Purity/0FRt1SRqqt2FBt/SRqqt_purity‚0.1538461595773697 =&Purity/*FRt*SRqqt1FBt/event_proportion‚0.0617094486951828 9"Purity/*FRt2SRqqt*FBt/SRqqt_purity‚0.6459209322929382 <&Purity/2FRt*SRqqt*FBt/event_proportion‚0.040062315762043 ,&Purity/2FRt2SRqqt1FBt/event_proportion‚0 7 Purity/0FRt1SRqqt2FBt/FBt_purity‚0.9230769276618958 7 Purity/*FRt0SRqqt1FBt/FBt_purity‚0.9981421232223511 9"Purity/0FRt*SRqqt0FBt/event_purity‚0.5964857935905457 8"Purity/0FRt0SRqqt*FBt/event_purity‚0.993191123008728 :"Purity/1FRt1SRqqt*FBt/event_purity‚0.03159944340586662 @&Purity/1FRt1SRqqt1FBt/event_proportion‚0.0064639137126505375 8 Purity/2FRt0SRqqt0FBt/FRt_purity‚0.13133451342582703 )jet/accuracy_2_of_4‚0.719189465045929 9"Purity/*FRt*SRqqt*FBt/event_purity‚0.2670508325099945 ("Purity/*FRt2SRqqt1FBt/event_purity‚0 7 Purity/0FRt1SRqqt*FBt/FBt_purity‚0.9913028478622437 >&Purity/1FRt*SRqqt1FBt/event_proportion‚0.01643880270421505 7 Purity/1FRt0SRqqt1FBt/FRt_purity‚0.6194849014282227 @&Purity/2FRt*SRqqt2FBt/event_proportion‚0.0002092633949359879 7 Purity/2FRt0SRqqt*FBt/FRt_purity‚0.1527484506368637 9"Purity/2FRt0SRqqt1FBt/event_purity‚0.2604166865348816 & Purity/1FRt2SRqqt*FBt/FBt_purity‚1 +jet/accuracy_1_of_1‚0.28991830348968506 particle/accuracy_1_of_5‚1 trainer/global_step‚5599 ,&Purity/*FRt2SRqqt2FBt/event_proportion‚0 9"Purity/2FRt2SRqqt0FBt/SRqqt_purity‚0.7622491121292114 7 Purity/*FRt1SRqqt1FBt/FBt_purity‚0.9987789392471313 9"Purity/0FRt2SRqqt0FBt/SRqqt_purity‚0.5925097465515137 8 Purity/1FRt1SRqqt*FBt/FRt_purity‚0.11657353490591049 =&Purity/2FRt0SRqqt*FBt/event_proportion‚0.0208565853536129 8 Purity/2FRt1SRqqt*FBt/FRt_purity‚0.17036685347557068 *jet/accuracy_1_of_4‚0.9471265077590942 8 Purity/2FRt1SRqqt0FBt/FRt_purity‚0.15041053295135498 7 Purity/*FRt*SRqqt1FBt/FBt_purity‚0.9980267882347107 8&Purity/*FRt2SRqqt0FBt/event_proportion‚ 0.05322265625 9"Purity/0FRt*SRqqt*FBt/event_purity‚0.6431729197502136 =&Purity/0FRt0SRqqt*FBt/event_proportion‚0.0390159972012043 >&Purity/0FRt2SRqqt0FBt/event_proportion‚0.02385602705180645 7 Purity/1FRt0SRqqt*FBt/FBt_purity‚0.9988385438919067 8 Purity/1FRt1SRqqt0FBt/FRt_purity‚0.09919049590826035 =&Purity/*FRt1SRqqt*FBt/event_proportion‚0.3866024911403656 & Purity/1FRt2SRqqt1FBt/FRt_purity‚0 =&Purity/2FRt*SRqqt0FBt/event_proportion‚0.0382486991584301 *"Purity/2FRt*SRqqt2FBt/event_purity‚0.5 =&Purity/*FRt0SRqqt0FBt/event_proportion‚0.4607747495174408 9"Purity/0FRt1SRqqt*FBt/SRqqt_purity‚0.6186209917068481 @&Purity/1FRt0SRqqt2FBt/event_proportion‚0.0024879092816263437 7 Purity/1FRt0SRqqt2FBt/FRt_purity‚0.8872448801994324 ;"Purity/2FRt0SRqqt*FBt/event_purity‚0.027902359142899513 )loss/val_total_loss‚9.644850333992977 9"Purity/0FRt2SRqqt*FBt/SRqqt_purity‚0.5868300199508667 :"Purity/1FRt2SRqqt1FBt/SRqqt_purity‚0.32499998807907104 particle/accuracy_1_of_3‚1 9"Purity/*FRt*SRqqt0FBt/event_purity‚0.2285788208246231 9"Purity/*FRt1SRqqt0FBt/SRqqt_purity‚0.6771634817123413 ,&Purity/0FRt2SRqqt2FBt/event_proportion‚0 9"Purity/1FRt1SRqqt2FBt/SRqqt_purity‚0.1111111<×n 119389534 >&Purity/1FRt2SRqqt*FBt/event_proportion‚0.02322823740541935 & Purity/2FRt*SRqqt*FBt/FBt_purity‚1 =&Purity/0FRt*SRqqt1FBt/event_proportion‚0.0436662957072258 @&Purity/2FRt1SRqqt2FBt/event_proportion‚2.325148852833081e-05 ("Purity/2FRt2SRqqt*FBt/event_purity‚0 ?&Purity/*FRt*SRqqt2FBt/event_proportion‚0.010997953824698925 =&Purity/*FRt0SRqqt1FBt/event_proportion‚0.0418526791036129 7 Purity/0FRt*SRqqt*FBt/FBt_purity‚0.9932601451873779 9"Purity/0FRt1SRqqt0FBt/event_purity‚0.6246905326843262 9"Purity/1FRt*SRqqt*FBt/event_purity‚0.1135038435459137 <"Purity/2FRt*SRqqt0FBt/event_purity‚0.0077096084132790565 ) Purity/2FRt*SRqqt2FBt/FRt_purity‚0.75 7 Purity/1FRt*SRqqt2FBt/FBt_purity‚0.9894558191299438 9"Purity/1FRt2SRqqt*FBt/SRqqt_purity‚0.6821696162223816 :"Purity/*FRt2SRqqt*FBt/event_purity‚0.15426039695739746 7 Purity/2FRt*SRqqt1FBt/FRt_purity‚0.5133333206176758 /particle/accuracy_3_of_3‚0.9994933605194092 jet/accuracy_1_of_5‚1 ?&Purity/2FRt1SRqqt0FBt/event_proportion‚0.011928013525903225 =&Purity/*FRt1SRqqt1FBt/event_proportion‚0.0188569575548172 ("Purity/1FRt2SRqqt0FBt/event_purity‚0 'particle/f_score‚0.6427584290504456 ("Purity/2FRt1SRqqt2FBt/event_purity‚0 =&Purity/2FRt1SRqqt*FBt/event_proportion‚0.0127883180975914 ("Purity/2FRt1SRqqt1FBt/event_purity‚0 =&Purity/*FRt0SRqqt*FBt/event_proportion‚0.5129278302192688 9"Purity/*FRt0SRqqt0FBt/event_purity‚0.1293083280324936 9"Purity/1FRt1SRqqt0FBt/SRqqt_purity‚0.7458270192146301 ,jet/accuracy_4_of_4‚0.013899499550461769 particle/accuracy_2_of_2‚1 6validation_average_jet_accuracy‚0.3381672501564026 & Purity/2FRt0SRqqt2FBt/FBt_purity‚1 9"Purity/*FRt1SRqqt1FBt/SRqqt_purity‚0.5750026702880859 9"Purity/*FRt2SRqqt0FBt/SRqqt_purity‚0.6510157585144043 +jet/accuracy_3_of_3‚0.06437167525291443 particle/accuracy_1_of_2‚1 9"Purity/1FRt*SRqqt1FBt/SRqqt_purity‚0.6158331632614136 ("Purity/1FRt1SRqqt2FBt/event_purity‚0 particle/accuracy_5_of_5‚0 9"Purity/*FRt0SRqqt2FBt/event_purity‚0.9197235107421875 ?&Purity/0FRt*SRqqt2FBt/event_proportion‚0.008068266324698925 @&Purity/1FRt*SRqqt2FBt/event_proportion‚0.0027204242069274187 :"Purity/1FRt1SRqqt1FBt/event_purity‚0.22856537997722626 & Purity/2FRt*SRqqt1FBt/FBt_purity‚1 =&Purity/0FRt1SRqqt0FBt/event_proportion‚0.2083565890789032 ("Purity/0FRt2SRqqt1FBt/event_purity‚0 9"Purity/1FRt0SRqqt2FBt/event_purity‚0.7789115905761719 7 Purity/1FRt1SRqqt2FBt/FBt_purity‚0.8888888955116272 >&Purity/2FRt0SRqqt0FBt/event_proportion‚0.01990327425301075 ("Purity/2FRt1SRqqt2FBt/SRqqt_purity‚0 & Purity/2FRt1SRqqt2FBt/FBt_purity‚1 7 Purity/0FRt*SRqqt1FBt/FBt_purity‚0.9971537590026855 A&Purity/1FRt2SRqqt1FBt/event_proportion‚0.00027901786961592734 ?&Purity/2FRt2SRqqt*FBt/event_proportion‚0.006417410913854837 /particle/accuracy_3_of_4‚0.9942177534103394 9"Purity/*FRt*SRqqt1FBt/SRqqt_purity‚0.5550576448440552 6 Purity/0FRt0SRqqt2FBt/FBt_purity‚0.987664520740509 7 Purity/*FRt*SRqqt*FBt/FBt_purity‚0.9944456815719604 @&Purity/0FRt2SRqqt1FBt/event_proportion‚0.0007207961170934141 & Purity/1FRt0SRqqt1FBt/FBt_purity‚1 % _timestamp‚1.7844642151305945e+09 7 Purity/*FRt0SRqqt1FBt/FRt_purity‚0.6058465242385864 7 Purity/*FRt1SRqqt*FBt/FBt_purity‚0.9920156598091125 9"Purity/0FRt*SRqqt1FBt/SRqqt_purity‚0.5136484503746033 9"Purity/2FRt*SRqqt*FBt/SRqqt_purity‚0.7550957798957825 7 Purity/*FRt*SRqqt2FBt/FRt_purity‚0.8795163035392761 9"Purity/0FRt1SRqqt1FBt/SRqqt_purity‚0.5291898846626282 @&Purity/0FRt1SRqqt2FBt/event_proportion‚0.0004417782765813172 =&Purity/1FRt1SRqqt0FBt/event_proportion‚0.1467633992433548 7 Purity/*FRt1SRqqt2FBt/FRt_purity‚0.8500000238418579 7 Purity/*FRt*SRqqt1FBt/FRt_purity‚0.5441954731941223ÍC‰/ǯ‚(:’ á†óÒ”ñ†¥ gpu.0.gpu‚44.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚50.0 gpu.0.powerWatts‚175.873' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚58.62433333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Y¾o°‚(:ä á†óÒÏà«¥" disk.dm-0.out‚55184.8388671875 disk.dm-1.in‚0 disk.rbd0.out‚0' disk.nvme1c1n1.out‚16862.5419921875% proc.memory.availableMB‚ 44.890625" proc.memory.rssMB‚ 2931.0390625 disk.md0.in‚0$ disk.nvme0n1.out‚40530.298828125 disk.nvme1c1n1.in‚ 129.765625( proc.memory.percent‚17.8896427154541 cpu‚25.9024943899646 proc.cpu.threads‚17% disk./.usageGB‚1172.5030937194824 disk.dm-0.in‚ 1982.2109375 disk.dm-1.out‚ 2003.05859375$ memory_percent‚99.72600936889648 network.recv‚ 1.036588e+06) disk./.usagePercent‚76.37212790877592% disk.nvme1n1.out‚16862.5419921875 disk.md0.out‚ 95.14453125 disk.rbd0.in‚0" disk.nvme0c0n1.in‚ 1852.4453125& disk.nvme0c0n1.out‚40530.298828125 disk.nvme0n1.in‚ 1852.4453125 network.sent‚ 8.381934e+06 disk.nvme1n1.in‚ 129.765625n¿^‰l±‚(:á ð†óÒ’ùõ¤ disk.nvme1c1n1.in‚ 129.765625% disk.nvme1n1.out‚16863.3193359375 disk.dm-0.in‚ 1982.2109375" disk.dm-0.out‚55745.9169921875 disk.rbd0.out‚0 disk.nvme0n1.in‚ 1852.4453125% proc.memory.availableMB‚ 44.453125 network.sent‚ 8.440898e+06 network.recv‚ 1.040979e+06) disk./.usagePercent‚76.37231724597288 disk.nvme1n1.in‚ 129.765625 disk.dm-1.in‚0 disk.rbd0.in‚0$ memory_percent‚99.72867965698242 cpu‚25.90128155470341% disk./.usageGB‚1172.5060005187988" disk.nvme0c0n1.in‚ 1852.4453125' disk.nvme1c1n1.out‚16863.3193359375 disk.md0.in‚0 disk.dm-1.out‚ 2003.28125" proc.memory.rssMB‚ 2931.0390625( proc.memory.percent‚17.8896427154541 proc.cpu.threads‚17 disk.md0.out‚ 95.5703125& disk.nvme0c0n1.out‚41091.744140625$ disk.nvme0n1.out‚41091.744140625g“²‚(:ˆ ð†óÒ®í§¥ gpu.0.gpu‚54.0 gpu.0.memory‚13, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚49.0 gpu.0.powerWatts‚218.169' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚72.723 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0Ã?f2³- _wandbruntime‚3922  _runtime‚3922ú¬”É´Ã trainer/global_step‚5649 % _timestamp‚1.7844642478520346e+09 _runtime‚3922.240351452 _step‚462 ' lr-AdamW/pg1‚0.00011602152741507717 ' lr-AdamW/pg2‚0.00011602152741507717e4 tÎµÈ trainer/global_step‚5649 % _timestamp‚1.7844642478520346e+09 _runtime‚3922.240351452 _step‚462 ' lr-AdamW/pg1‚0.00011602152741507717 ' lr-AdamW/pg2‚0.00011602152741507717Î_€”¹2¶- _wandbruntime‚3922  _runtime‚3922ÚÕ°&å'·ßO JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚438565622998528 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚56844490 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚42881082 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚333270750187008 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4650429768157 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚56844490 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚42881258 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 101667656 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚443228822118400 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚30083267 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚443228927817728 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048U)yëê'¸äO YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚438565605959168 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚30083271 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚438038857360557 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 _step‚463 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚56854633 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚12797815 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 114763776 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4663304819200 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚176 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 208873472 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4663216159232 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚44813021 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 % _timestamp‚1.7844642478527427e+09 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚172 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 101657511 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚443228927817728 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚333270959060480 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚438038961186343 _runtime‚ 3922.24484352 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚12797987 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚438565622998528 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚56844490 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚42881082 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚333270750187008 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4650429768157 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚56844490 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚42881258 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 101667656 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚443228822118400 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚30083267 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚443228927817728 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 JDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚180 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚42888527 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4801029814784 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚333327908472832 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚438640772664832 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12800020 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚56864467 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚12800200 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚438640755625472 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 101685119 MCDeviceStatsMonitor.on_train_batch_eŽGl\nd/allocation.small_pool.current‚10157 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚438113931380549 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 101685119 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚443304863206912 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4664090542080 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚44820652 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚56854310 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚333327701703680 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 101674960 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚44820652 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚443304863206912 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4651213262757 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚438640772664832 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4664001874944 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚44820650 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚328526671888896 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 trainer/global_step‚5649 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4664001874944 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚438114035206447 _step‚464 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚44820650 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 112659456 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚42888343 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚56864467 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚328526765998592 _runtime‚3922.647597743'õ&»ïM UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚180 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚42888527 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4801029814784 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚333327908472832 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚438640772664832 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚12800020 EDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚56864467 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚12800200 BDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚438640755625472 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 F8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 101685119 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚438113931380549 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 101685119 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚443304863206912 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4664090542080 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚44820652 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚56854310 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚333327701703680 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 101674960 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚44820652 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚443304863206912 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4651213262757 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚438640772664832 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4664001874944 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚44820650 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚328526671888896 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 trainer/global_step‚5649 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4664001874944 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚438114035206447 _step‚464 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚44820650 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 112659456 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚42888343 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚56864467 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚328526765998592 _runtime‚3922.647597743 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚56854310 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚30088323 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496Ð&µ²Ý2¼- _wandbruntime‚3922  _runtime‚3922#Óè ½ƒ /loss/FRt2/detection_loss‚1.3375499803723225 2loss/FBt1/assignment_loss‚0.023692171884235693 2loss/FBt2/assignment_loss‚0.042578263153426284 1loss/SRqqt2/detection_loss‚0.6119328852113933 %loss/total_loss‚9.807479142348516 epoch‚14 _step‚465 1loss/SRqqt1/detection_loss‚1.0534666966944932 /loss/FBt1/detection_loss‚0.5113255326614613 .loss/FBt2/detection_loss‚1.088987931322593 _runtime‚3922.649692987 /loss/FRt1/assignment_loss‚2.352835742748931 0loss/FRt2/assignment_loss‚1.0561420767326055 2loss/SRqqt1/assignment_loss‚0.6269857486085966 2loss/SRqqt2/assignment_loss‚0.6055225950928617 0loss/FRt1/detection_loss‚0.49645951786559517 trainer/global_step‚5649 % _timestamp‚1.7844642482607744e+09š¤Î¾ˆ 0loss/FRt1/detection_loss‚0.49645951786559517 trainer/global_step‚5649 % _timestamp‚1.7844642482607744e+09 /loss/FRt2/detection_loss‚1.3375499803723225 2loss/FBt1/assignment_loss‚0.023692171884235693 2loss/FBt2/assignment_loss‚0.042578263153426284 1loss/SRqqt2/detection_loss‚0.6119328852113933 %loss/total_loss‚9.807479142348516 epoch‚14 _step‚465 1loss/SRqqt1/detection_loss‚1.0534666966944932 /loss/FBt1/detection_loss‚0.5113255326614613 .loss/FBt2/detection_loss‚1.088987931322593 _runtime‚3922.649692987 /loss/FRt1/assignment_loss‚2.352835742748931 0loss/FRt2/assignment_loss‚1.0561420767326055 2loss/SRqqt1/assignment_loss‚0.6269857486085966 2loss/SRqqt2/assignment_loss‚0.6055225950928617ÑÛ¹QŸ¿‚(:” ÿ†óÒÀ½ƒ¥ gpu.0.gpu‚91.0 gpu.0.memory‚19, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚49.0 gpu.0.powerWatts‚114.644' gpu.0.enforcedPowerLimitWatts‚300.0) gpu.0.powerPercent‚38.214666666666666 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¼DlÝkÀ‚(:à ÿ†óÒѸ”§& proc.memory.availableMB‚ 44.3984375 disk.nvme1n1.out‚ 16865.5625 disk.dm-0.in‚ 1982.2109375# disk.dm-0.out‚55764.43408203125" disk.nvme0c0n1.in‚ 1852.4453125 disk.nvme0n1.in‚ 1852.4453125" proc.memory.rssMB‚ 2931.0390625( proc.memory.percent‚17.8896427154541 network.sent‚ 8.456764e+06 network.recv‚ 1.04427e+06 disk.dm-1.in‚0 disk.rbd0.out‚0 cpu‚25.900865462237014 proc.cpu.threads‚17$ disk./.usageGB‚1172.506175994873 disk.nvme1n1.in‚ 129.765625 disk.md0.out‚ 95.80859375 disk.nvme1c1n1.in‚ 129.765625$ memory_percent‚99.72901344299316) disk./.usagePercent‚76.37232867577744 disk.md0.in‚0! disk.dm-1.out‚2003.4365234375 disk.rbd0.in‚0( disk.nvme0c0n1.out‚41108.70068359375& disk.nvme0n1.out‚41108.70068359375! disk.nvme1c1n1.out‚ 16865.5625wÄ‹‡qÁ‚(:æ Ž‡óÒ¥Úˤ network.recv‚ 1.04869e+06) disk./.usagePercent‚76.37232668798534 disk.rbd0.in‚0' disk.nvme1c1n1.out‚16866.0009765625 disk.md0.out‚ 96.0859375 disk.dm-0.in‚ 1982.2109375" proc.memory.availableMB‚44.625 network.sent‚ 8.506239e+06" disk.nvme0c0n1.in‚ 1852.4453125( disk.nvme0c0n1.out‚41112.38525390625 disk.nvme0n1.in‚ 1852.4453125 disk.md0.in‚0" proc.memory.rssMB‚ 2931.0390625( proc.memory.percent‚17.8896427154541 cpu‚25.90064479750756 disk.dm-1.in‚0 disk.nvme1n1.in‚ 129.765625% disk.nvme1n1.out‚16866.0009765625 proc.cpu.threads‚17 disk.nvme1c1n1.in‚ 129.765625$ disk./.usageGB‚1172.506145477295! disk.dm-1.out‚2003.4755859375 disk.rbd0.out‚0& disk.nvme0n1.out‚41112.38525390625# disk.dm-0.out‚55767.89892578125$ memory_percent‚99.72763061523438ÞU2AœÂ‚(:‘ Ž‡óÒýï¥ gpu.0.gpu‚90.0 gpu.0.memory‚20, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚49.0 gpu.0.powerWatts‚142.3' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚47.43333333333334 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0M(P2Ã- _wandbruntime‚3954  _runtime‚3954*úSfÈÄ ' lr-AdamW/pg1‚0.00011560406014477845 ' lr-AdamW/pg2‚0.00011560406014477845 trainer/global_step‚5699 $ _timestamp‚1.784464280388917e+09 _runtime‚3954.778813395 _step‚466àëÍÅÇ ' lr-AdamW/pg1‚0.00011560406014477845 ' lr-AdamW/pg2‚0.00011560406014477845 trainer/global_step‚5699 $ _timestamp‚1.784464280388917e+09 _runtime‚3954.778813395 _step‚466Òѱ¯2Æ- _wandbruntime‚3954  _runtime‚3954{<(_æ'ÇàO KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 215164928 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 151689728 % _timestamp‚1.7844642803896344e+09 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚447025761073152 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 102541958 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚336118561699328 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4689668298965 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚447025655373824 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚182 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚447025655373824 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚57347485 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4840925973504 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚442323089274368 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚30336071 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2©Ö×^ ÈåO P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚12908176 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4702654759424 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚43244429 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚57337342 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4841047028736 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚441792725986751 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚442323089274368 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚45194473 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 _step‚467 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4702566099456 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚437102970901360 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4689755085391 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚441792622160965 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚45194473 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 [KDxœ\äeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 215164928 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 151689728 % _timestamp‚1.7844642803896344e+09 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚447025761073152 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 102541958 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚336118561699328 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4689668298965 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚447025655373824 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚182 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚447025655373824 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚57347485 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4840925973504 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚442323089274368 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚30336071 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 JDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚442398255980032 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚442398238940672 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚45202100 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚437177261426752 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚43251648 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚45202102 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚43251450 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚441867800006855 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚57347162 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚45202100 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚442398255980032 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 217254912 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4703351815168 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4703440482304 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 102559421 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚442398238940672 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 102549262 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚441867696180957 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚447101696462336 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚194 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4841864103424 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚447101696462336 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 102559421 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚336175299247104 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚57347162 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚198 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚30341123 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚12910521 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 _step‚468 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚336175516502016 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚331333558288896 % _timestamp‚1.7844642808036432e+09 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚447101590755840 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚45202102 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚437177244387392 trainer/global_step‚5699 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4703440482304 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚331333652398592 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 151689728 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4690451793565 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 102549262 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0¨­nnõ&ËïM MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚30341123 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚12910521 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 _step‚468 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚336175516502016 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚331333558288896 % _timestamp‚1.7844642808036432e+09 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚447101590755840 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚45202102 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚437177244387392 trainer/global_step‚5699 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4703440482304 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚331333652398592 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 151689728 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4690451793565 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 102549262 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚442398255980032 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚442398238940672 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚45202100 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚437177261426752 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚43251648 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚45202102 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚43251450 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚441867800006855 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚57347162 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚45202100 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚442398255980032 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 217254912 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4703351815168 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4703440482304 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 102559421 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚442398238940672 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 102549262 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚441867696180957 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚447101696462336 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚194 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4841864103424 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚447101696462336 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 102559421 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚336175299247104 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚57347162 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚198 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 EDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0Ôxéù2Ì- _wandbruntime‚3955  _runtime‚3955Û¢ ÇÍ 0loss/FRt1/assignment_loss‚2.3411509263335244 2loss/SRqqt2/assignment_loss‚0.5517913017078395 1loss/FBt1/assignment_loss‚0.09606334123976902 /loss/FBt1/detection_loss‚0.3693877545254607 trainer/global_step‚5699 1loss/FBt2/assignment_loss‚0.07845626411025454 /loss/FRt2/detection_loss‚1.4663144469960816 1loss/SRqqt1/detection_loss‚1.1916519080197239 .loss/FBt2/detection_loss‚1.011300126694747 %loss/total_loss‚9.908890959595713 % _timestamp‚1.7844642808051534e+09 _step‚469 2loss/SRqqt1/assignment_loss‚0.5960309473874597 _runtime‚3955.193359668 0loss/FRt2/assignment_loss‚1.0568828468048383 /loss/FRt1/detection_loss‚0.5303590323815733 1loss/SRqqt2/detection_loss‚0.6195020633944427 epoch‚14bͳ˜ Ά 0loss/FRt2/assignment_loss‚1.0568828468048383 /loss/FRt1/detection_loss‚0.5303590323815733 1loss/SRqqt2/detection_loss‚0.6195020633944427 epoch‚14 0loss/FRt1/assignment_loss‚2.3411509263335244 2loss/SRqqt2/assignment_loss‚0.5517913017078395 1loss/FBt1/assignment_loss‚0.09606334123976902 /loss/FBt1/detection_loss‚0.3693877545254607 trainer/global_step‚5699 1loss/FBt2/assignment_loss‚0.07845626411025454 /loss/FRt2/detection_loss‚1.4663144469960816 1loss/SRqqt1/detection_loss‚1.1916519080197239 .loss/FBt2/detection_loss‚1.011300126694747 %loss/total_loss‚9.908890959595713 % _timestamp‚1.7844642808051534e+09 _step‚469 2loss/SRqqt1/assignment_loss‚0.5960309473874597 _runtime‚3955.193359668ÕÃÝæ(rÏ‚(:ç ‡óÒáø̤) disk./.usagePercent‚76.37233712389383 disk.rbd0.in‚0 disk.nvme1c1n1.in‚ 129.765625 disk.nvme1n1.in‚ 129.765625 disk.dm-1.in‚0" disk.nvme0c0n1.in‚ 1852.4453125( disk.nvme0c0n1.out‚41114.12646484375& disk.nvme0n1.out‚41114.12646484375' disk.nvme1c1n1.out‚16866.5576171875% disk.nvme1n1.out‚16866.5576171875 disk.md0.in‚0! disk.dm-1.out‚2003.5751953125& proc.memory.availableMB‚ 44.3046875# disk./.usageGB‚1172.50630569458 disk.rbd0.out‚0 disk.dm-0.in‚ 1982.2109375$ memory_percent‚99.72958564758301( proc.memory.percent‚17.8896427154541 cpu‚25.899824302553256 network.sent‚ 8.5122e+06 network.recv‚ 1.051808e+06 disk.nvme0n1.in‚ 1852.4453125 disk.md0.out‚96.34375# disk.dm-0.out‚55769.53466796875" proc.memory.rssMB‚ 2931.0390625 proc.cpu.threads‚17•b_Ђ(:’ ‡óÒ¥›¾¤ gpu.0.gpu‚46.0 gpu.0.memory‚7, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚50.0 gpu.0.powerWatts‚226.552' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚75.51733333333334 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0â@WÊwÑ‚(:ì ¬‡óÒ˜ÙÚ¤ network.recv‚ 1.056243e+06 disk.md0.out‚ 97.0078125 disk.dm-0.in‚ 1982.2109375# disk.dm-0.out‚55770.34326171875! disk.dm-1.out‚2003.6689453125 disk.rbd0.in‚0" disk.nvme0c0n1.in‚ 1852.4453125 cpu‚25.898569878519737 network.sent‚ 8.575497e+06$ disk./.usageGB‚1172.506061553955 disk.md0.in‚0 disk.nvme1c1n1.in‚ 129.765625 disk.nvme1n1.in‚ 129.765625% disk.nvme1n1.out‚16867.4990234375 disk.dm-1.in‚0 disk.rbd0.out‚0( disk.nvme0c0n1.out‚41115.49365234375& proc.memory.availableMB‚ 44.5078125( proc.memory.percent‚17.8896427154541 proc.cpu.threads‚17) disk./.usagePercent‚76.37232122155709 disk.nvme0n1.in‚ 1852.4453125& disk.nvme0n1.out‚41115.49365234375' disk.nvme1c1n1.out‚16867.4990234375$ memory_percent‚99.72834587097168" proc.memory.rssMB‚ 2931.0390625\²!“Ò‚(:ˆ ¬‡óÒʈŒ¥ gpu.0.gpu‚80.0 gpu.0.memory‚19, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚50.0 gpu.0.powerWatts‚220.572' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚73.524 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0>~$2Ó- _wandbruntime‚3989  _runtime‚3989L~¯ÇÔÁ trainer/global_step‚5749 # _timestamp‚1.78446431464175e+09 _runtime‚3989.030806011 _step‚470 ' lr-AdamW/pg1‚0.00011518363948741995 ' lr-AdamW/pg2‚0.00011518363948741995ÚSöÌÕÆ # _timestamp‚1.78446431464175e+09 _runtime‚3989.030806011 _step‚470 ' lr-AdamW/pg1‚0.00011518363948741995 ' lr-AdamW/pg2‚0.00011518363948741995 trainer/global_step‚5749Ö¢çÂ2Ö- _wandbruntime‚3989  _runtime‚3989ÌýèÄ@×ßO TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚13016597 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚334084306960896 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4728906829773 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0a©_¥$ @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚45575921 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚446080572589568 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚446080589628928 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚446080572589568 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚450822488629248 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 151689728 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚445546490787159 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚450822594328576 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚338965902420480 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4881595459584 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚45575921 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 _runtime‚3989.030873352 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚45575923 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚446080589628928 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚440817480131600 trainer/global_step‚5749 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4728993616199 _step‚471 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4741916039680 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚57840337 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 127346688 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚445546386961373 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚30588871 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚193 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚440817497170960 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚43605271 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 221456384 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚30588867 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4742004699648 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚57840337 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚43605468 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚197 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4881722806272 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4742004699648 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚13016404 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚334084401070592 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 221456384 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚30588867 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4742004699648 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚57840337 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚43605468 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚197 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 4881722806272 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4742004699648 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚13016404 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚334084401070592 BDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚450822488629248 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 151689728 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚445546490787159 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚450822594328576 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚338965902420480 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4881595459584×GKy2Ù- _wandbruntime‚3989  _runtime‚3989NCÉ$ð&ÚêM UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚45583552 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4742701755392 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4742790422528 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚445621564807263 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4742790422528 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚450898424011264 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚43612446 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚339023074452480 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4729777110911 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚30593923 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚187 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4882412508672 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 103433723 trainer/global_step‚5749 F6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚13018710 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚45583552 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚191 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚339022857197568 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 151689728 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚445621460981365 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚57840014 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚450898529717760 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚334140444688896 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚446155739295232 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _runtime‚3989.450534359 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚57850171 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 103433723 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚57850171 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚30593927 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0šý K, ÛïM REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚13018523 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 217254912 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚440891770656992 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚446155739295232 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4729690324373 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚45583550 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4882535653888 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚13018710 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 E6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚45583552 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚191 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚339022857197568 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 151689728 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚4456´çZ~É21460981365 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚57840014 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚450898529717760 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚334140444688896 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚446155739295232 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 _runtime‚3989.450534359 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚57850171 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 103433723 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚57850171 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚30593927 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚45583552 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4742701755392 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4742790422528 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚445621564807263 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4742790422528 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚450898424011264 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 BDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚43612446 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚339023074452480 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4729777110911 PDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚30593923 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚187 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4882412508672 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 103433723 trainer/global_step‚5749 FDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4781265979904 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4922145592832 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚449300151761781 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4768232147007 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 _step‚475 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚449838055904768 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚43964983 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚45957373 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚454619321884672 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚58323046 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚43965171 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 JDeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145‚T‡%êåO L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚58323046 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 trainer/global_step‚5799 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚30841671 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 151689728 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 123152384 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚13123316 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚454619427584000 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚449838072944128 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4781265979904 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 104290562 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚336891193360896 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚45957371 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4781265979904 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 4922145592832 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚449300151761781 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4768232147007 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 _step‚475 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚449838055904768 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚43964983 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚45957373 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚454619321884672 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚58323046 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚43965171 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 JDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚13125551 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 _step‚476 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚30846727 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 _runtime‚ 4022.62349887 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4923085156352 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 104308025 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4782140362752 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚43972096 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚454695362973184 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚13125373 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4768928855181 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚341870291002880 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4782051695616 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 $ _timestamp‚1.784464348234956e+09 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚444606313965952 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚449913222610432 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 125242368 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 104297866 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚449913205571072 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚454695257266688 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 104297866 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚45965002 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚449913222610432 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚336947331088896 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 FDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚178 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4769015641719 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚444606296926592 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚58332866 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚449913205571072 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 104308025 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 219352064 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚454695362973184 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4922959913984 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚449375329607671 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚449375225781773 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4782140362752 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4782051695616 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 trainer/global_step‚5799 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚58332866 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚45965002WwÄöó&ííM N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚178 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4769015641719 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚444606296926592 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚58332866 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚449913205571072 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 104308025 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 219352064 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚454695362973184 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4922959913984 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚449375329607671 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚449375225781773 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4782140362752 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4782051695616 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 trainer/global_step‚5799 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚58332866 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚45965002 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚45965000 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚182 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 151689728 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚341870510354944 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚13125551 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 _step‚476 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚30846727 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 _runtime‚ 4022.62349887 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4923085156352 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 104308025 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4782140362752 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚43972096 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚454695362973184 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚13125373 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4768928855181 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚341870291002880 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4782051695616 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 $ _timestamp‚1.784464348234956e+09 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚444606313965952 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚449913222610432 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 125242368 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 104297866 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚449913205571072 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚454695257266688 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 104297866 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚45965002 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚449913222610432 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚336947331088896 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 FDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 _runtime‚4055.070187158 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚458416155140096 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4820615920128 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚44324773 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚31094471 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚339698079760896 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚458416260839424 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 105164864 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚344660979800576 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚46338823 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4820615920128 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚453595539219968 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚58826041 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚448246549710160 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚344660764635648 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4820704580096 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚453595556259328 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚458416155140096 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 121055232 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚458416260839424 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚46338821 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4807470677815 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚453595556259328oÝÁÈê'úäO P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4820704580096 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚453595556259328 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚458416155140096 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 121055232 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚458416260839424 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚46338821 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4807470677815 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚453595556259328 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚58815898 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 _runtime‚4055.070187158 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚458416155140096 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4820615920128 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚44324773 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚31094471 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚339698079760896 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚458416260839424 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 105164864 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚344660979800576 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚46338823 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4820615920128 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚453595539219968 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚58826041 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚448246549710160 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚344660764635648 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0ßõæÛ2û- _wandbruntime‚4055  _runtime‚40553eVï&üéM PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 105182327 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚44332012 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚58825718 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚13232485 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚13232308 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚453128990582181 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚458492196228608 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚344717705412608 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4963623651840 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚453670705925632 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4808167385989 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚339754311598592 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 105172168 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4821490302976 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚448320823196192 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚458492196228608 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚44331831 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4821490302976 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚31099523 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚344717935250432 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 229837824 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚46346452 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 trainer/global_step‚5849 _step‚480 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4821401635840 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 135728128 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚58835875 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚458492090522112 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚58835875 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚453670688886272 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚453670705925632 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 105182327 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚177 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚46346450 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚339754217488896 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4963487923712 _runtime‚4055.483148265 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚181 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 151689728 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4808254172527 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚453670688886272 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚46346450 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚58825718 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 $ _timestamp‚1.784464381094631e+09 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚46346452 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚31099527 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0¼÷Nö %ýîM WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚453670688886272 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚46346450 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚58825718 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 $ _timestamp‚1.784464381094631e+09 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚46346452 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚31099527 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 105182327 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚44332012 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚58825718 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated‚13232485 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚13232308 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚453128990582181 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚458492196228608 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚344717705412608 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 4963623651840 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚453670705925632 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4808167385989 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚339754311598592 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 105172168 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4821490302976 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚448320823196192 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚458492196228608 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚44331831 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4821490302976 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚31099523 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚344717935250432 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 229837824 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚46346452 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 trainer/global_step‚5849 _step‚480 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4821401635840 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 135728128 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚58835875 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚458492090522112 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚58835875 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚453670688886272 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current‚17039360 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚453670705925632 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 105182327 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚177 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚46346450 P8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚339754217488896 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 4963487923712 _runtime‚4055.483148265 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚181 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 151689728 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4808254172527 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 FDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0àlM2þ- _wandbruntime‚4055  _runtime‚4055²¶V~ÿþ 1loss/FBt2/assignment_loss‚0.04575161252975511 .loss/FRt2/detection_loss‚1.391942288156408 1loss/SRqqt1/detection_loss‚1.1376314149773183 /loss/FBt2/detection_loss‚0.9355987429929277 epoch‚14 _runtime‚4055.483672536 %loss/total_loss‚9.901411874878882 $ _timestamp‚1.784464381095635e+09 _step‚481 /loss/FRt1/assignment_loss‚2.270043196014522 2loss/SRqqt2/assignment_loss‚0.5798449309565588 trainer/global_step‚5849 2loss/SRqqt1/assignment_loss‚0.7181993088650299 /loss/FRt1/detection_loss‚0.5190758718578604 1loss/SRqqt2/detection_loss‚0.6797923543261962 /loss/FBt1/detection_loss‚0.5570486837717257 /loss/FRt2/assignment_loss‚1.020373810464003 1loss/FBt1/assignment_loss‚0.04610965996657661C{®¡ €ƒ /loss/FRt2/assignment_loss‚1.020373810464003 1loss/FBt1/assignment_loss‚0.04610965996657661 1loss/FBt2/assignment_loss‚0.04575161252975511 .loss/FRt2/detection_loss‚1.391942288156408 1loss/SRqqt1/detection_loss‚1.1376314149773183 /loss/FBt2/detection_loss‚0.9355987429929277 epoch‚14 _runtime‚4055.483672536 %loss/total_loss‚9.901411874878882 $ _timestamp‚1.784464381095635e+09 _step‚481 /loss/FRt1/assignment_loss‚2.270043196014522 2loss/SRqqt2/assignment_loss‚0.5798449309565588 trainer/global_step‚5849 2loss/SRqqt1/assignment_loss‚0.7181993088650299 /loss/FRt1/detection_loss‚0.5190758718578604 1loss/SRqqt2/detection_loss‚0.6797923543261962 /loss/FBt1/detection_loss‚0.5570486837717257áÀÌj)r‚(:ç †ˆóÒðË’¥" disk.nvme0c0n1.in‚ 1852.4453125 network.recv‚ 1.079195e+06% disk./.usageGB‚1169.3620071411133 disk.dm-0.in‚ 1982.2109375" disk.dm-0.out‚55780.8681640625" disk.dm-1.out‚2004.17724609375& disk.nvme0n1.out‚41127.03857421875% disk.nvme1c1n1.out‚16871.24609375 disk.md0.in‚0 disk.md0.out‚ 98.9765625 disk.rbd0.in‚0$ memory_percent‚99.72352981567383( proc.memory.percent‚17.8896427154541 cpu‚25.90166050693513 proc.cpu.threads‚17 disk.rbd0.out‚0 network.sent‚ 8.731927e+06) disk./.usagePercent‚76.16753018343027 disk.dm-1.in‚0 disk.nvme0n1.in‚ 1852.4453125% proc.memory.availableMB‚ 45.296875( disk.nvme0c0n1.out‚41127.03857421875 disk.nvme1c1n1.in‚ 129.765625 disk.nvme1n1.in‚ 129.765625# disk.nvme1n1.out‚16871.24609375" proc.memory.rssMB‚ 2931.0390625n)ÿ5‚‚(:’ †ˆóÒÇ’Æ¥ gpu.0.gpu‚36.0 gpu.0.memory‚9, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚48.0 gpu.0.powerWatts‚114.599' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚38.19966666666667 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0ÓÙsßlƒ‚(:á •ˆóÒ½ªË¤ disk.rbd0.in‚0" disk.nvme0c0n1.in‚ 1852.4453125$ memory_percent‚99.72505569458008" proc.memory.rssMB‚ 2931.0390625) disk./.usagePercent‚76.16755527930546 disk.md0.in‚0 disk.rbd0.out‚0' disk.nvme0c0n1.out‚41134.1162109375 disk.nvme0n1.in‚ 1852.4453125% disk.nvme0n1.out‚41134.1162109375 proc.cpu.threads‚17 disk.dm-1.in‚0 network.sent‚ 8.782277e+06 network.recv‚ 1.083386e+06' disk.nvme1c1n1.out‚16873.0517578125! disk.dm-0.out‚55789.048828125% proc.memory.availableMB‚ 45.046875 cpu‚25.90331076856369$ disk./.usageGB‚1169.362392425537 disk.dm-0.in‚ 1982.2109375 disk.dm-1.out‚ 2004.265625( proc.memory.percent‚17.8896427154541 disk.nvme1c1n1.in‚ 129.765625 disk.nvme1n1.in‚ 129.765625% disk.nvme1n1.out‚16873.0517578125 disk.md0.out‚ 99.265625æý™až„‚(:“ •ˆóÒ Çý¤ gpu.0.gpu‚88.0 gpu.0.memory‚15, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚49.0 gpu.0.powerWatts‚138.127' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚46.04233333333333 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0cd÷2…- _wandbruntime‚4088  _runtime‚4088y—žÉ†Ã _runtime‚4088.218353961 _step‚482 ' lr-AdamW/pg1‚0.00011390492815935252 ' lr-AdamW/pg2‚0.00011390492815935252 trainer/global_step‚5899 % _timestamp‚1.7844644138299875e+093§ÓÎ‡È ' lr-AdamW/pg1‚0.00011390492815935252 ' lr-AdamW/pg2‚0.00011390492815935252 trainer/global_step‚5899 % _timestamp‚1.7844644138299875e+09 _runtime‚4088.218353961 _step‚482â‚—(‰2ˆ- _wandbruntime‚4088  _runtime‚4088zEjëå'‰ßO RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚462213094094848 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚44683380 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5003358018560 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚457353022535168 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚195 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4860054520320 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 131540992 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 106029021 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4859965860352 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚462213094094848 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚59318893 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4846709208623 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚457353022535168 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚462212988395520 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚13336109 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚46720271 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚451961058940400 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚457353039574528 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 151689728 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4860054520320 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚59308750 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚31347271 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5003226477568 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚191 _runtime‚ 4088.22285842 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚44683185 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚451961075979760 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚342505060270592 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 JDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4859965860352 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 106039166 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 trainer/global_step‚5899 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚342504966160896 % _timestamp‚1.7844644138309395e+09 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚462212988395520 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0¼»®×ê'ŠäO TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚31347267 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 BDeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4859965860352 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 106039166 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current‚86786426 trainer/global_step‚5899 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚342504966160896 % _timestamp‚1.7844644138309395e+09 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚462212988395520 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚462213094094848 ICDeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated‚0 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 QDDeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated‚44683380 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5003358018560 ?9DeviceStatsMonitor.on_train_batch_start/num_alloc_retries‚0 ?9DeviceStatsMonitor.on_train_batch_start/segment.all.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚457353022535168 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚195 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4860054520320 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 131540992 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 106029021 WEDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed‚ 4859965860352 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚462213094094848 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚59318893 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4846709208623 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 LDDeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated‚682 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚457353022535168 JDeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚462212988395520 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚13336109 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 H>DeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚46720271 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚451961058940400 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚457353039574528 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 151689728 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4860054520320 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚59308750 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚31347271 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5003226477568 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚191 _runtime‚ 4088.22285842 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚44683185 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚451961075979760 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 A7DeviceStatsMonitor.on_train_batch_start/active.all.peak‚12374 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚342505060270592 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 JDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4860840243200 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚46727902 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚342561197998592 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚457428172201472 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚462289029484032 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4860840243200 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚59328727 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚452035349465792 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 106046470 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚46727902 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 151689728 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚44690443 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚172 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 106056629 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4847405916797 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 106056629 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚59328727 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 106046470 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚46727900 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚342561103888896 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WIDeviceStatsMonitor.on_trainHžsÆ _batch_end/reserved_bytes.small_pool.allocated‚ 272629760 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚347565143909376 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚457428172201472 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚59318570 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5004040020480 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚31352327 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5004169457152 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚44690271 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚456882859208487 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚452035366505152 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 _step‚4848ºõ&ïM E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 UGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current‚ 272629760 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 trainer/global_step‚5899 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚46727900 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚462289029484032 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 _runtime‚4088.631391236 PDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 \JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated‚ 4860840243200 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚46727902 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚342561197998592 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚457428172201472 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚462289029484032 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4860840243200 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚59328727 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚452035349465792 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 106046470 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚46727902 FDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 ZMDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current‚94109696 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 151689728 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚44690443 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚172 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 106056629 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4847405916797 FDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 106056629 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚59328727 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 106046470 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚46727900 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚342561103888896 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚347565143909376 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚457428172201472 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚59318570 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5004040020480 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚31352327 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5004169457152 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 K>DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚44690271 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚456882859208487 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚452035366505152 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 _step‚484 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚456882755382589 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682äjy2Ž- _wandbruntime‚4088  _runtime‚4088uL %loss/total_loss‚9.279661328186371 0loss/FRt1/assignment_loss‚2.1475049714784364 /loss/FRt2/detection_loss‚1.4502892385393047 /loss/FBt1/detection_loss‚0.3869796692319383 _step‚485 2loss/SRqqt1/assignment_loss‚0.5806673947663512 1loss/SRqqt2/detection_loss‚0.6714069721216877 .loss/FBt2/detection_loss‚0.863566453935522 0loss/FRt2/assignment_loss‚0.9978424888140509 2loss/SRqqt2/assignment_loss‚0.5402043143172388 1loss/FBt2/assignment_loss‚0.09260702375025841 /loss/FRt1/detection_loss‚0.5634683559085141 1loss/SRqqt1/detection_loss‚0.9284040620734569 trainer/global_step‚5899 _runtime‚4088.632157088 1loss/FBt1/assignment_loss‚0.05672038324961193 epoch‚14 % _timestamp‚1.7844644142439332e+09jE¿ † 0loss/FRt2/assignment_loss‚0.9978424888140509 2loss/SRqqt2/assignment_loss‚0.5402043143172388 1loss/FBt2/assignment_loss‚0.09260702375025841 /loss/FRt1/detection_loss‚0.5634683559085141 1loss/SRqqt1/detection_loss‚0.9284040620734569 trainer/global_step‚5899 _runtime‚4088.632157088 1loss/FBt1/assignment_loss‚0.05672038324961193 epoch‚14 % _timestamp‚1.7844644142439332e+09 %loss/total_loss‚9.279661328186371 0loss/FRt1/assignment_loss‚2.1475049714784364 /loss/FRt2/detection_loss‚1.4502892385393047 /loss/FBt1/detection_loss‚0.3869796692319383 _step‚485 2loss/SRqqt1/assignment_loss‚0.5806673947663512 1loss/SRqqt2/detection_loss‚0.6714069721216877 .loss/FBt2/detection_loss‚0.863566453935522åÂDÃèr‘‚(:ç ¤ˆóÒ”¥€¥ disk.md0.in‚0 disk.dm-0.in‚ 1982.2109375 disk.dm-1.in‚0 disk.dm-1.out‚ 2004.3828125 disk.rbd0.out‚0' disk.nvme0c0n1.out‚41135.1650390625 disk.nvme1c1n1.in‚ 129.765625 disk.nvme1n1.in‚ 129.765625 network.sent‚ 8.798084e+06 network.recv‚ 1.087075e+06% disk./.usageGB‚1169.3624076843262 disk.md0.out‚ 99.76953125' disk.nvme1c1n1.out‚16873.8154296875% disk.nvme1n1.out‚16873.8154296875$ memory_percent‚99.72383975982666" proc.memory.rssMB‚ 2931.0390625 disk.rbd0.in‚0" disk.nvme0c0n1.in‚ 1852.4453125% disk.nvme0n1.out‚41135.1650390625' proc.memory.availableMB‚ 45.24609375( proc.memory.percent‚17.8896427154541 cpu‚25.90505788105851 proc.cpu.threads‚17) disk./.usagePercent‚76.16755627320151! disk.dm-0.out‚55789.693359375 disk.nvme0n1.in‚ 1852.4453125™ŒÞˆ’’‚(:‡ ¤ˆóÒÙ¬¶¥ gpu.0.gpu‚47.0 gpu.0.memory‚8, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚50.0 gpu.0.powerWatts‚220.224' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚73.408 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0äiÝJm“‚(:â ³ˆóÒŸñð¤ network.sent‚ 8.84946e+06% disk./.usageGB‚1169.3627967834473& disk.nvme0c0n1.out‚41138.462890625$ disk.nvme0n1.out‚41138.462890625 disk.dm-0.in‚ 1982.2109375! disk.dm-0.out‚55793.009765625" proc.memory.rssMB‚ 2931.0390625( proc.memory.percent‚17.8896427154541 network.recv‚ 1.091117e+06) disk./.usagePercent‚76.16758161755071 disk.nvme0n1.in‚ 1852.4453125 disk.md0.in‚0 disk.md0.out‚ 100.1953125$ memory_percent‚99.72381591796875! proc.memory.availableMB‚45.25 proc.cpu.threads‚17 disk.nvme1n1.in‚ 129.765625% disk.nvme1n1.out‚16874.8154296875 disk.rbd0.in‚0" disk.nvme0c0n1.in‚ 1852.4453125 disk.nvme1c1n1.in‚ 129.765625' disk.nvme1c1n1.out‚16874.8154296875! disk.dm-1.out‚2004.4482421875 disk.rbd0.out‚0 disk.dm-1.in‚0 cpu‚25.906678054578883ê±ož”‚(:“ ³ˆóÒÚù¤¥ gpu.0.gpu‚68.0 gpu.0.memory‚16, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚49.0 gpu.0.powerWatts‚218.647' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚72.88233333333332 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0[ˆÃ2•- _wandbruntime‚4121  _runtime‚4121UâgFÆ–À _runtime‚ 4121.32677863 _step‚486 & lr-AdamW/pg1‚0.0001134729658529049 & lr-AdamW/pg2‚0.0001134729658529049 trainer/global_step‚5949 % _timestamp‚1.7844644469379973e+09ºúrúË—Å % _timestamp‚1.7844644469379973e+09 _runtime‚ 4121.32677863 _step‚486 & lr-AdamW/pg1‚0.0001134729658529049 & lr-AdamW/pg2‚0.0001134729658529049 trainer/global_step‚5949æÙðĽ2˜- _wandbruntime‚4121  _runtime‚4121ÕXÜ4æ'™àO `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚461110522889728 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚466009821650944 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚31600071 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚460561446163005 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 227747840 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚460561549988791 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚191 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 106913468 BDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚47101721 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚461110522889728 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚466009927350272 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚47101723 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚59801602 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4899404460544 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚47101721 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚59811745 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚13441179 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚345311946670592 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚455675602249360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚466009927350272 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5043825529344 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚31600067 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5043691891200 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚59801602 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚345311852560896 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 151689728 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4885860953005 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚466009821650944 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚350355772199936 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4885947739431 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚455675585210000 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚59811745 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚187 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚461110505850368 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4899404460544 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812¼Fd\kšåO WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 G>DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚47101721 ]IDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated‚461110522889728 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚466009927350272 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚47101723 PCDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed‚59801602 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4899404460544 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚47101721 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 JDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚59811745 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚13441179 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚345311946670592 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚455675602249360 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak‚ 272629760 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.¸)07€peak‚130 VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚466009927350272 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5043825529344 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 [KDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated‚ 14365491200 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚31600067 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5043691891200 L?DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed‚59801602 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚345311852560896 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 151689728 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 ZHDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed‚ 4885860953005 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚466009821650944 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚350355772199936 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 GADeviceStatsMonitor.on_train_batch_start/oversize_segments.current‚0 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4885947739431 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 \HDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed‚455675585210000 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚59811745 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚187 GADeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak‚0 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚461110505850368 LBDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak‚12303 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4899404460544 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 E=DeviceStatsMonitor.on_train_batch_start/segment.all.allocated‚812 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚461110522889728 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚466009821650944 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚31600071 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚460561446163005 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 WGDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak‚ 11862167552 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 VHDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current‚ 227747840 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚460561549988791 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚191 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 N@DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated‚ 106913468 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 106930931 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚191 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚13443020 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4886731234143 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚59821579 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚455749892774752 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚461185672556032 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚47109352 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚59821579 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚45048334 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5044495595520 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚31605123 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 _step‚488 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚45048143 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4886644447605 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 $ _timestamp‚1.784464447351939e+09 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4900101516288 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4900190183424 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚187 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 151689728 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚59811422 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚350412485884416 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚345368084398592 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 133630976 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 227740672 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 trainer/global_step‚5949 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 106930931 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚47109350 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 _runtime‚4121.740505892 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚59811422 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚345367990288896 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚47109352 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚466085757032960 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚31605127 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚350412713625088 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 106920772 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4900101516288 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚461185672556032 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚466085862739456 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚460636520182997 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5044629226496 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚460636624008895 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚466085862739456 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 106920772 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 B6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak‚350 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 227740672 JBDeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated‚682 trainer/global_step‚5949 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 106930931 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚47109350 =7DeviceStatsMonitor.on_train_batch_end/num_alloc_retries‚0 _runtime‚4121.740505892 E6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 D>DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚59811422 _KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed‚345367990288896 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚47109352 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚466085757032960 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚31605127 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚350412713625088 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 106920772 REDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current‚17039360 UCDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed‚ 4900101516288 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚461185672556032 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚466085862739456 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚460636520182997 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 K;DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak‚ 12301877248 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5044629226496 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚460636624008895 T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚466085862739456 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 >8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 M=DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak‚ 14638120960 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 H:DeviceStatsMonitor.on_train_batch_end/allocation.all.freed‚ 106920772 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 BDeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 106930931 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 ;5DeviceStatsMonitor.on_train_batch_end/num_device_free‚0 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚191 REDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed‚13443020 \JDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated‚ 4886731234143 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚59821579 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚455749892774752 ^JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated‚461185672556032 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚47109352 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚59821579 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚45048334 ]KDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed‚ 5044495595520 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 SEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak‚ 269300736 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚31605123 LDDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak‚486 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 _step‚488 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚45048143 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4886644447605 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 $ _timestamp‚1.784464447351939e+09 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4900101516288 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 YIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated‚ 14365491200 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4900190183424 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 E=DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak‚821 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚187 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 151689728 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 A9DeviceStatsMonitor.on_train_batch_end/segment.all.current‚812 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚59811422 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚350412485884416 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 MGDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current‚4 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚345368084398592 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 133630976 HBDeviceStatsMonitor.on_train_batch_end/oversize_allocations.current‚0è1ö(2ž- _wandbruntime‚4121  _runtime‚4121ßá­Ÿ‚ _step‚489 /loss/FRt1/detection_loss‚0.5794599331976048 /loss/FBt2/detection_loss‚0.6951819773721981 _runtime‚4121.741108173 0loss/FRt2/assignment_loss‚0.9450366958885116 2loss/SRqqt1/assignment_loss‚0.6815875925066179 /loss/FRt2/detection_loss‚1.5647714585816757 1loss/SRqqt1/detection_loss‚0.9637111938146659 %loss/total_loss‚9.678645047473072 epoch‚14 2loss/FBt1/assignment_loss‚0.051700459994131684 1loss/FBt2/assignment_loss‚0.21738696125084125 1loss/SRqqt2/detection_loss‚0.6382676944607494 $ _timestamp‚1.784464447352988e+09 0loss/FRt1/assignment_loss‚2.1776148953167995 2loss/SRqqt2/assignment_loss‚0.6496133313439397 /loss/FBt1/detection_loss‚0.5143128537453364 trainer/global_step‚59497àwì  ‡ 2loss/FBt1/assignment_loss‚0.051700459994131684 1loss/FBt2/assignment_loss‚0.21738696125084125 1loss/SRqqt2/detection_loss‚0.6382676944607494 $ _timestamp‚1.784464447352988e+09 0loss/FRt1/assignment_loss‚2.1776148953167995 2loss/SRqqt2/assignment_loss‚0.6496133313439397 /loss/FBt1/detection_loss‚0.5143128537453364 trainer/global_step‚5949 _step‚489 /loss/FRt1/detection_loss‚0.5794599331976048 /loss/FBt2/detection_loss‚0.6951819773721981 _runtime‚4121.741108173 0loss/FRt2/assignment_loss‚0.9450366958885116 2loss/SRqqt1/assignment_loss‚0.6815875925066179 /loss/FRt2/detection_loss‚1.5647714585816757 1loss/SRqqt1/detection_loss‚0.9637111938146659 %loss/total_loss‚9.678645047473072 epoch‚14éÚoÇz¡‚(:ï ˆóÒ¼ê¥ disk.md0.in‚0 disk.md0.out‚ 100.73046875 disk.dm-0.in‚ 1982.2109375 disk.nvme1n1.in‚ 129.765625$ memory_percent‚99.72519874572754" proc.memory.rssMB‚ 2931.0390625) disk./.usagePercent‚76.29811669420951" disk.nvme0c0n1.in‚ 1852.4453125& disk.nvme0n1.out‚41167.88525390625 proc.cpu.threads‚17 network.recv‚ 1.094477e+06! disk.dm-0.out‚55821.974609375! disk.dm-1.out‚2004.4873046875 disk.rbd0.out‚0 disk.nvme0n1.in‚ 1852.4453125 disk.nvme1c1n1.in‚ 129.765625( disk.nvme1c1n1.out‚16875.53857421875( proc.memory.percent‚17.8896427154541% disk./.usageGB‚1171.3668365478516 disk.dm-1.in‚0 disk.rbd0.in‚0& disk.nvme1n1.out‚16875.53857421875& proc.memory.availableMB‚ 45.0234375 cpu‚25.907617429453058( disk.nvme0c0n1.out‚41167.88525390625 network.sent‚ 8.865326e+06¾“)z“¢‚(:ˆ ˆóÒõ£Ê¥ gpu.0.gpu‚90.0 gpu.0.memory‚16, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚49.0 gpu.0.powerWatts‚122.049' gpu.0.enforcedPowerLimitWatts‚300.0 gpu.0.powerPercent‚40.683 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0äz3r£‚(:ç шóÒöÂÞ¤ disk.dm-0.out‚57710.64453125 network.recv‚ 1.098564e+06% disk./.usageGB‚1172.1236877441406" disk.nvme0c0n1.in‚ 1852.4453125 disk.md0.out‚ 101.0390625 disk.dm-1.in‚0 disk.rbd0.in‚0( proc.memory.percent‚17.8896427154541 cpu‚25.906330743863727 network.sent‚ 8.91698e+06 disk.nvme0n1.in‚ 1852.4453125 disk.nvme1c1n1.in‚ 129.765625 disk.dm-0.in‚ 1982.2109375 disk.rbd0.out‚0' proc.memory.availableMB‚ 44.99609375" proc.memory.rssMB‚ 2931.0390625! disk.dm-1.out‚2004.6611328125$ memory_percent‚99.72536563873291 proc.cpu.threads‚17$ disk.nvme1n1.out‚16876.337890625 disk.md0.in‚0) disk./.usagePercent‚76.34741493203977' disk.nvme0c0n1.out‚43056.6103515625% disk.nvme0n1.out‚43056.6103515625& disk.nvme1c1n1.out‚16876.337890625 disk.nvme1n1.in‚ 129.765625iÁºJž¤‚(:“ шóÒÔ¥ gpu.0.gpu‚90.0 gpu.0.memory‚19, gpu.0.memoryAllocated‚17.681655883789062* gpu.0.memoryAllocatedBytes‚ 15188426752 gpu.0.temp‚48.0 gpu.0.powerWatts‚132.293' gpu.0.enforcedPowerLimitWatts‚300.0( gpu.0.powerPercent‚44.09766666666667 gpu.0.smClock‚1410 gpu.0.memoryClock‚1512! gpu.0.correctedMemoryErrors‚0# gpu.0.uncorrectedMemoryErrors‚0¢a_³2¥- _wandbruntime‚4153  _runtime‚4153q®M“ǦÁ & lr-AdamW/pg1‚0.0001130381870356016 & lr-AdamW/pg2‚0.0001130381870356016 trainer/global_step‚5999 % _timestamp‚1.7844644792974665e+09 _runtime‚4153.686656577 _step‚490KÑÌ§Æ & lr-AdamW/pg1‚0.0001130381870356016 & lr-AdamW/pg2‚0.0001130381870356016 trainer/global_step‚5999 % _timestamp‚1.7844644792974665e+09 _runtime‚4153.686656577 _step‚490ê •Í2¨- _wandbruntime‚4153  _runtime‚4153¸åb  ©àO VBDeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated‚469806760605696 _step‚491 UADeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed‚469806654906368 \HDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed‚464867989165568 ZHDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed‚ 4938665740800 TGDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated‚60304597 HBDeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed‚0 VFDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak‚ 14365491200 E;DeviceStatsMonitor.on_train_batch_start/allocation.all.peak‚12374 cQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated‚ 5084255289344 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚464867989165568 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚47483173 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚353202860612096 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚47483171 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚464868006204928 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚47483171 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚469806654906368 trainer/global_step‚5999 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚60304597 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak[ÙBF‚ 269300736 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚183 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚353203088359936 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4938754400768 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚469806760605696 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚459390128518960 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚31852871 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚348118833070592 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 133638144 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4925186270239 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚464315314789199 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚31852867 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4938754400768 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚13546108 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚47483173 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚13546291 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 107777625 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚464315210963413 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚348118738960896 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5084121651200 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚187 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚45398975 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 151689728 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 JDeviceStatsMonitor.on_train_batch_start/active.large_pool.peak‚1509 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current‚17039360 YEDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed‚464867989165568 QCDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current‚ 105699328 TGDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated‚47483173 ZFDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed‚353202860612096 TDDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated‚ 14638120960 RBDeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current‚ 14638120960 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak‚350 PCDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed‚47483171 G?DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak‚682 `LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated‚464868006204928 UGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak‚ 269300736 KEDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current‚2 \ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current‚94109696 WJDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current‚17039360 C;DeviceStatsMonitor.on_train_batch_start/segment.all.current‚812 @8DeviceStatsMonitor.on_train_batch_start/segment.all.peak‚812 L?DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed‚47483171 R>DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed‚469806654906368 trainer/global_step‚5999 PCDeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated‚60304597 RDDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak‚ 269300736 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current‚17039360 JDDeviceStatsMonitor.on_train_batch_start/oversize_allocations.current‚0 D>DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak‚0 LDDeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated‚130 TGDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current‚88659968 QIDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current‚183 ^JDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated‚353203088359936 =7DeviceStatsMonitor.on_train_batch_start/num_device_free‚0 E?DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed‚0 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_start/segment.small_pool.current‚130 F@DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed‚0 KADeviceStatsMonitor.on_train_batch_start/active.small_pool.current‚10143 [IDeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated‚ 4938754400768 YEDeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated‚469806760605696 =6DeviceStatsMonitor.on_train_batch_start/max_split_size‚-1 `LDeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated‚459390128518960 O?DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak‚ 14638120960 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated‚31852871 NFDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak‚486 @8DeviceStatsMonitor.on_train_batch_start/num_device_alloc‚812 BDeviceStatsMonitor.on_train_batch_start/allocation.all.current‚10145 JBDeviceStatsMonitor.on_train_batch_start/segment.large_pool.current‚682 M=DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak‚ 12301877248 G?DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak‚821 eQDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated‚348118833070592 ]ODeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current‚ 133638144 ^LDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated‚ 4925186270239 UGDeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak‚ 267417279 YEDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated‚464315314789199 [LDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak‚ 2342719488 MGDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed‚0 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed‚31852867 GADeviceStatsMonitor.on_train_batch_start/active.large_pool.current‚2 ^LDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated‚ 4938754400768 OEDeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current‚10143 TGDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed‚13546108 TEDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak‚ 2372707840 PCDeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated‚47483173 XKDeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated‚13546291 F8DeviceStatsMonitor.on_train_batch_start/active.all.freed‚ 107777625 TDDeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak‚ 12065026048 LFDeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated‚0 UADeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed‚464315210963413 P@DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak‚ 12106884175 WIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current‚ 272629760 YKDeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated‚ 272629760 G?DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak‚130 WJDeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current‚88659968 N@DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current‚ 105699328 P@DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak‚ 12301877248 aMDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed‚348118738960896 _MDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed‚ 5084121651200 @:DeviceStatsMonitor.on_train_batch_start/num_oom_rejections‚0 F@DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed‚0 D:DeviceStatsMonitor.on_train_batch_start/active.all.current‚10145 KBDeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak‚1509 JBDeviceStatsMonitor.on_train_batch_start/inactive_split.all.current‚187 M@DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed‚45398975 ZLDeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak‚ 151689728 F@DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed‚0 OIDeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current‚4 YIDeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current‚ 14365491200 H>DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak‚12303 WGDeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak‚ 12065026048 60DeviceStatsMonitor.on_train_batch_start/num_ooms‚0 QCDeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current‚ 103825786 JDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak‚ 12301877248 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4939540123648 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚353260035257344 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚31857923 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚464390388809303 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5085064458752 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 _step‚492 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 229837824 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚459464402004992 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚464943138831872 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 107805233 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚464943138831872 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚31857927 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4925882978413 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4939451456512 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚353259805419520 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 EDeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚45406097 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚464943155871232 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚348174970798592 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚459464419044352 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚47490802 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚47490800 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚464390284983405 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚189 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚60304274 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚193 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 151689728 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 % _timestamp‚1.7844644797129984e+09 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 107795074 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚47490800 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚45406290 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 135728128 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 _runtime‚4154.101679948 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚60314431 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚60314431 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚469882695994880 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 trainer/global_step‚5999 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚47490802œƒöŠ­ïM T@DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated‚469882695994880 NADeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed‚60304274 OADeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current‚ 103825898 D>DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed‚0 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed‚0 PDeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed‚45406097 E=DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak‚682 [GDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated‚464943155871232 UEDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak‚ 12065026048 cODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated‚348174970798592 F@DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed‚0 C=DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed‚0 ^JDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated‚459464419044352 B8DeviceStatsMonitor.on_train_batch_end/active.all.current‚10159 NADeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated‚47490802 J=DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed‚47490800 S?DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed‚464390284983405 C;DeviceStatsMonitor.on_train_batch_end/segment.all.allocated‚812 E?DeviceStatsMonitor.on_train_batch_end/active.large_pool.current‚2 RBDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak‚ 12065026048 C9DeviceStatsMonitor.on_train_batch_end/allocation.all.peak‚12374 YJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak‚ 2342719488 JDDeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated‚0 PBDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak‚ 269300736 OGDeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current‚189 RDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak‚ 272629760 J=DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed‚60304274 H@DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current‚193 XJDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak‚ 151689728 E?DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current‚86786538 WGDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current‚ 14365491200 % _timestamp‚1.7844644797129984e+09 D6DeviceStatsMonitor.on_train_batch_end/active.all.freed‚ 107795074 NADeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed‚47490800 OBDeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated‚45406290 [MDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current‚ 135728128 >6DeviceStatsMonitor.on_train_batch_end/num_device_alloc‚812 _runtime‚4154.101679948 NADeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated‚60314431 UHDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current‚88667136 FDeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed‚0 JBDeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated‚130 REDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated‚60314431 WCDeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated‚469882695994880 TDDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak‚ 14365491200 WIDeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated‚ 272629760 trainer/global_step‚5999 I?DeviceStatsMonitor.on_train_batch_end/active.small_pool.current‚10157 REDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated‚47490802 RBDeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated‚ 14638120960 H:DeviceStatsMonitor.on_train_batch_end/active.all.allocated‚ 107805233 S?DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed‚469882590288384 N>DeviceStatsMonitor.on_train_batch_end/allocated_bytes.allõqÆúk.peak‚ 12301877248 SEDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak‚ 267417279 P@DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current‚ 14638120960 =7DeviceStatsMonitor.on_train_batch_end/segment.all.freed‚0 MCDeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current‚10157 REDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current‚88667136 YGDeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated‚ 4939540123648 I@DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak‚1509 J@DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak‚12303 \HDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated‚353260035257344 N>DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak‚ 12106884175 UEDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak‚ 11862167552 H@DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current‚130 ICDeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current‚2 REDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed‚31857923 WCDeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated‚464390388809303 >6DeviceStatsMonitor.on_train_batch_end/segment.all.peak‚812 RCDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak‚ 2372707840 aODeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated‚ 5085064458752 @:DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams‚2 _step‚492 OADeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current‚ 105706496 TFDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current‚ 229837824 ZFDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed‚459464402004992 L>DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current‚ 105706496 ZFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed‚464943138831872 L>DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated‚ 107805233 GADeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated‚0 D>DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed‚0 H@DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current‚682 ?5DeviceStatsMonitor.on_train_batch_end/active.all.peak‚12374 4.DeviceStatsMonitor.on_train_batch_end/num_ooms‚0 E?DeviceStatsMonitor.on_train_batch_end/oversize_segments.current‚0 UHDeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current‚17039360 KEDeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed‚0 WCDeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed‚464943138831872 VIDeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated‚31857927 B8DeviceStatsMonitor.on_train_batch_end/num_oom_rejections‚0 XFDeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed‚ 4925882978413 E=DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak‚130 XFDeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed‚ 4939451456512 XDDeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed‚353259805419520 ;4DeviceStatsMonitor.on_train_batch_end/max_split_size‚-1 E´ÿ{ ;"Purity/2FRt*SRqqt*FBt/event_purity‚0.015844153240323067 ?&Purity/2FRt1SRqqt0FBt/event_proportion‚0.011928013525903225 ,&Purity/2FRt2SRqqt2FBt/event_proportion‚0 particle/accuracy_5_of_5‚0 7 Purity/0FRt1SRqqt1FBt/FBt_purity‚0.9959307312965393 & Purity/1FRt2SRqqt*FBt/FBt_purity‚1 jet/accuracy_5_of_5‚0 =&Purity/1FRt0SRqqt0FBt/event_proportion‚0.4408714771270752 +jet/accuracy_2_of_2‚0.12939508259296417 7 Purity/*FRt0SRqqt2FBt/FBt_purity‚0.9878005981445312 6 Purity/*FRt1SRqqt*FBt/FBt_purity‚0.990800142288208 8"Purity/*FRt2SRqqt1FBt/SRqqt_purity‚0.381944477558136 :"Purity/1FRt0SRqqt*FBt/event_purity‚0.14021824300289154 >&Purity/2FRt0SRqqt0FBt/event_proportion‚0.01990327425301075 & Purity/2FRt1SRqqt1FBt/FBt_purity‚1 =&Purity/*FRt0SRqqt*FBt/event_proportion‚0.5129278302192688 8 Purity/2FRt1SRqqt0FBt/FRt_purity‚0.16379518806934357 >&Purity/1FRt0SRqqt1FBt/event_proportion‚0.00969587080180645 *jet/accuracy_1_of_2‚0.7732095122337341 % _timestamp‚1.7844644931283498e+09 7 Purity/1FRt*SRqqt2FBt/FRt_purity‚0.8565760254859924 =&Purity/1FRt1SRqqt0FBt/event_proportion‚0.1467633992433548 8 Purity/1FRt2SRqqt1FBt/FRt_purity‚0.10000000149011612 /particle/accuracy_3_of_3‚0.9994933605194092 particle/accuracy_1_of_4‚1 8"Purity/1FRt2SRqqt*FBt/SRqqt_purity‚0.677079975605011 ("Purity/2FRt1SRqqt1FBt/event_purity‚0 7 Purity/0FRt*SRqqt*FBt/FBt_purity‚0.9921701550483704 7 Purity/1FRt*SRqqt2FBt/FBt_purity‚0.9894558191299438 7 Purity/1FRt0SRqqt*FBt/FBt_purity‚0.9988385438919067 8 Purity/1FRt1SRqqt1FBt/FRt_purity‚0.44262850284576416 ("Purity/1FRt2SRqqt1FBt/event_purity‚0 7 Purity/2FRt*SRqqt1FBt/FRt_purity‚0.5358333587646484 ;"Purity/2FRt0SRqqt0FBt/event_purity‚0.020245129242539406 8 Purity/2FRt1SRqqt*FBt/FRt_purity‚0.18487510085105896 7 Purity/*FRt0SRqqt2FBt/FRt_purity‚0.8468632698059082 =&Purity/0FRt*SRqqt1FBt/event_proportion‚0.0436662957072258 9"Purity/0FRt2SRqqt0FBt/SRqqt_purity‚0.5888653993606567 ,&Purity/2FRt2SRqqt1FBt/event_proportion‚0 9"Purity/0FRt*SRqqt0FBt/SRqqt_purity‚0.6178103685379028 =&Purity/0FRt0SRqqt*FBt/event_proportion‚0.0390159972012043 9"Purity/1FRt1SRqqt*FBt/SRqqt_purity‚0.7467018365859985 <&Purity/2FRt*SRqqt*FBt/event_proportion‚0.040062315762043 7 Purity/*FRt0SRqqt1FBt/FBt_purity‚0.9981421232223511 =&Purity/0FRt0SRqqt1FBt/event_proportion‚0.0313895083963871 9"Purity/0FRt1SRqqt0FBt/SRqqt_purity‚0.6245434284210205 @&Purity/0FRt1SRqqt2FBt/event_proportion‚0.0004417782765813172 >&Purity/0FRt2SRqqt*FBt/event_proportion‚0.02457682229578495 @&Purity/0FRt2SRqqt1FBt/event_proportion‚0.0007207961170934141 7 Purity/1FRt*SRqqt*FBt/FRt_purity‚0.1296166181564331 @&Purity/2FRt1SRqqt2FBt/event_proportion‚2.325148852833081e-05 8 Purity/*FRt0SRqqt0FBt/FRt_purity‚0.12736909091472626 =&Purity/0FRt1SRqqt0FBt/event_proportion‚0.2083565890789032 & Purity/2FRt1SRqqt2FBt/FBt_purity‚1 :"Purity/*FRt1SRqqt1FBt/event_purity‚0.40683332085609436 :"Purity/*FRt2SRqqt*FBt/event_purity‚0.15227782726287842 9"Purity/0FRt1SRqqt*FBt/SRqqt_purity‚0.6196860074996948 >&Purity/1FRt2SRqqt*FBt/event_proportion‚0.02322823740541935 & Purity/2FRt0SRqqt*FBt/FBt_purity‚1 @&Purity/2FRt*SRqqt1FBt/event_proportion‚0.0016043527284637094 @&Purity/2FRt1SRqqt1FBt/event_proportion‚0.0008370535797439516 =&Purity/*FRt*SRqqt*FBt/event_proportion‚0.9537528157234192 =&Purity/*FRt1SRqqt*FBt/event_proportion‚0.3866024911403656 =&Purity/1FRt*SRqqt0FBt/event_proportion‚0.6105840802192688 8 Purity/1FRt*SRqqt0FBt/FRt_purity‚0.11510719358921051 :"Purity/1FRt1SRqqt0FBt/event_purity‚0.02252895012497902 9"Purity/2FRt2SRqqt*FBt/SRqqt_purity‚0.7490408420562744 particle/accuracy_2_of_2‚1 (particle/accuracy‚0.8498379588127136 8 Purity/*FRt0SRqqt*FBt/FRt_purity‚0.14258141815662384 & Purity/1FRt*SRqqt1FBt/FBt_purity‚1 *jet/accuracy_1_of_4‚0.9491969347000122 9"Purity/2FRt*SRqqt1FBt/event_purity‚0.0950000062584877 9"Purity/2FRt1SRqqt*FBt/SRqqt_purity‚0.7434011697769165 ( Purity/2FRt1SRqqt2FBt/FRt_purity‚0.5 trainer/global_step‚5999 7 Purity/*FRt*SRqqt1FBt/FRt_purity‚0.5424790382385254 9"Purity/2FRt0SRqqt1FBt/event_purity‚0.2083333432674408 7 Purity/*FRt2SRqqt1FBt/FBt_purity‚0.9814814329147339 7 Purity/0FRt0SRqqt1FBt/FBt_purity‚0.9974045753479004 =&Purity/1FRt0SRqqt*FBt/event_proportion‚0.4530552327632904 particle/accuracy_2_of_4‚1 ("Purity/2FRt1SRqqt2FBt/event_purity‚0 9"Purity/*FRt1SRqqt*FBt/SRqqt_purity‚0.6741831302642822 8 Purity/*FRt2SRqqt0FBt/FRt_purity‚0.07300971448421478 ."Purity/0FRt2SRqqt1FBt/SRqqt_purity‚0.40625 9"Purity/1FRt*SRqqt*FBt/SRqqt_purity‚0.7306381464004517 ("Purity/1FRt1SRqqt2FBt/event_purity‚0 8"Purity/1FRt2SRqqt0FBt/SRqqt_purity‚0.681550920009613 6 Purity/*FRt*SRqqt*FBt/FBt_purity‚0.993619441986084 9"Purity/0FRt*SRqqt1FBt/SRqqt_purity‚0.5393243432044983 9"Purity/1FRt0SRqqt1FBt/event_purity‚0.5834558010101318 ;"Purity/2FRt*SRqqt0FBt/event_purity‚0.010464348830282688 <&Purity/*FRt2SRqqt*FBt/event_proportion‚0.054222472012043 =&Purity/0FRt0SRqqt0FBt/event_proportion‚0.0462472103536129 , Purity/0FRt2SRqqt1FBt/FBt_purity‚0.96875 9"Purity/*FRt0SRqqt1FBt/event_purity‚0.8855803608894348 9"Purity/0FRt*SRqqt*FBt/event_purity‚0.6443917155265808 ,&Purity/1FRt2SRqqt2FBt/event_proportion‚0 5 Purity/*FRt2SRqqt*FBt/FRt_purity‚0.07302226126194 9"Purity/0FRt2SRqqt*FBt/SRqqt_purity‚0.5835638046264648 ,&Purity/0FRt2SRqqt2FBt/event_proportion‚0 ("Purity/2FRt1SRqqt0FBt/event_purity‚0 ("Purity/2FRt2SRqqt0FBt/event_purity‚0 particle/accuracy_1_of_3‚1 :"Purity/*FRt*SRqqt*FBt/event_purity‚0.26439911127090454 9"Purity/*FRt0SRqqt2FBt/event_purity‚0.9150582551956177 =&Purity/*FRt1SRqqt1FBt/event_proportion‚0.0188569575548172 9"Purity/0FRt0SRqqt1FBt/event_purity‚0.9974045753479004 9"Purity/0FRt1SRqqt1FBt/event_purity‚0.5165648460388184 :"Purity/1FRt*SRqqt1FBt/event_purity‚0.45168453454971313 9"Purity/1FRt1SRqqt2FBt/SRqqt_purity‚0.1111111119389534 7 Purity/0FRt1SRqqt2FBt/FBt_purity‚0.9230769276618958 9"Purity/*FRt2SRqqt0FBt/SRqqt_purity‚0.6471351981163025 7 Purity/2FRt0SRqqt1FBt/FRt_purity‚0.5885416269302368 ("Purity/2FRt1SRqqt2FBt/SRqqt_purity‚0 particle/accuracy_3_of_5‚1 9"Purity/*FRt*SRqqt*FBt/SRqqt_purity‚0.6671638488769531 :"Purity/*FRt0SRqqt0FBt/event_purity‚0.12221173942089081 9"Purity/0FRt1SRqqt1FBt/SRqqt_purity‚0.5571209192276001 9"Purity/1FRt*SRqqt1FBt/SRqqt_purity‚0.6540589928627014 =&Purity/1FRt1SRqqt*FBt/event_proportion‚0.1534598171710968 & Purity/1FRt1SRqqt1FBt/FBt_purity‚1 8&Purity/1FRt2SRqqt0FBt/event_proportion‚ 0.02294921875 & Purity/2FRt*SRqqt1FBt/FBt_purity‚1 >&Purity/1FRt*SRqqt1FBt/event_proportion‚0.01643880270421505 8 Purity/1FRt1SRqqt0FBt/FRt_purity‚0.08636563271284103 8 Purity/2FRt*SRqqt0FBt/FRt_purity‚0.13310836255550385 @&Purity/2FRt*SRqqt2FBt/event_proportion‚0.0002092633949359879 ?&Purity/2FRt2SRqqt0FBt/event_proportion‚0.006417410913854837 jet/accuracy_3_of_5‚1 particle/accuracy_1_of_5‚1 particle/accuracy_2_of_5‚1 7 Purity/*FRt*SRqqt2FBt/FBt_purity‚0.9822051525115967 =&Purity/*FRt0SRqqt2FBt/event_proportion‚0.0103004090487957 8"Purity/0FRt*SRqqt*FBt/SRqqt_purity‚0.612989068031311 9"Purity/1FRt1SRqqt0FBt/SRqqt_purity‚0.7509251832962036 A&Purity/2FRt0SRqqt2FBt/event_proportion‚0.00018601190822664648 particle/accuracy_1_of_1‚1 *particle/specificity‚0.907795786857605 _step‚494 :"Purity/0FRt2SRqqt*FBt/event_purity‚0.33906635642051697 8 Purity/1FRt2SRqqt*FBt/FRt_purity‚0.07167483866214752 +jet/accuracy_1_of_1‚0.28660327196121216 6 Purity/*FRt*SRqqt2FBt/FRt_purity‚0.843461811542511 9"Purity/0FRt*SRqqt1FBt/event_purity‚0.8508739471435547 =&Purity/*FRt1SRqqt0FBt/event_proportion‚0.3670479953289032 8 Purity/1FRt1SRqqt*FBt/FRt_purity‚0.10316571593284607 7 Purity/2FRt*SRqqt*FBt/FRt_purity‚0.1519196480512619 8 Purity/2FRt1SRqqt1FBt/FRt_purity‚0.48842594027519226 particle/accuracy_2_of_3‚1 7 Purity/0FRt1SRqqt*FBt/FBt_purity‚0.9894083142280579 9"Purity/0FRt1SRqqt0FBt/event_purity‚0.6245434284210205 9"Purity/1FRt0SRqqt2FBt/event_purity‚0.7826153039932251 & Purity/1FRt2SRqqt1FBt/FBt_purity‚1 =&Purity/*FRt0SRqqt0FBt/event_proportion‚0.4607747495174408 :"Purity/*FRt1SRqqt*FBt/event_purity‚0.36516764760017395 ("Purity/0FRt2SRqqt1FBt/event_purity‚0 :"Purity/1FRt*SRqqt*FBt/event_purity‚0.10889540612697601 7 Purity/1FRt2SRqqt0FBt/FRt_purity‚0.0717899352312088 +jet/accuracy_3_of_3‚0.06693118810653687 jet/accuracy_4_of_5‚0 7 Purity/*FRt1SRqqt1FBt/FBt_purity‚0.9974919557571411 =&Purity/1FRt*SRqqt*FBt/event_proportion‚0.6297432780265808 8 Purity/1FRt0SRqqt*FBt/FRt_purity‚0.14144790172576904 @&Purity/1FRt1SRqqt1FBt/event_proportion‚0.0064639137126505375 9"Purity/2FRt*SRqqt1FBt/SRqqt_purity‚0.7777777910232544 ("Purity/2FRt1SRqqt*FBt/event_purity‚0 & Purity/2FRt1SRqqt*FBt/FBt_purity‚1 7 Purity/*FRt1SRqqt0FBt/FRt_purity‚0.0972537100315094 :"Purity/*FRt2SRqqt0FBt/event_purity‚0.15519270300865173 =&Purity/2FRt1SRqqt*FBt/event_proportion‚0.0127883180975914 @&Purity/*FRt1SRqqt2FBt/event_proportion‚0.0006975446594879031 ("Purity/2FRt2SRqqt*FBt/event_purity‚0 ?&Purity/2FRt2SRqqt*FBt/event_proportion‚0.006417410913854837 *jet/accuracy_1_of_3‚0.9059290885925293 ,jet/accuracy_4_of_4‚0.013899499550461769 & Purity/2FRt0SRqqt2FBt/FBt_purity‚1 particle/accuracy_4_of_5‚1 9"Purity/2FRt1SRqqt0FBt/SRqqt_purity‚0.7423023581504822 9"Purity/2FRt2SRqqt0FBt/SRqqt_purity‚0.7490408420562744 9"Purity/2FRt0SRqqt2FBt/event_purity‚0.4285714328289032 ?&Purity/*FRt*SRqqt2FBt/event_proportion‚0.010997953824698925 9"Purity/*FRt1SRqqt1FBt/SRqqt_purity‚0.6044234037399292 9"Purity/*FRt2SRqqt*FBt/SRqqt_purity‚0.6422409415245056 9"Purity/0FRt1SRqqt*FBt/event_purity‚0.6172688603401184 :"Purity/1FRt0SRqqt0FBt/event_purity‚0.12686528265476227 8 Purity/1FRt0SRqqt0FBt/FRt_purity‚0.12686528265476227 9"Purity/2FRt*SRqqt0FBt/SRqqt_purity‚0.7450370788574219 7 Purity/2FRt0SRqqt2FBt/FRt_purity‚0.7142857313156128 8 Purity/*FRt1SRqqt*FBt/FRt_purity‚0.11490442603826523 7 Purity/0FRt*SRqqt1FBt/FBt_purity‚0.9966744184494019 & Purity/1FRt0SRqqt1FBt/FBt_purity‚1 ;"Purity/2FRt0SRqqt*FBt/event_purity‚0.030597344040870667 +particle/sensitivity‚0.6378941535949707 jet/accuracy_1_of_5‚1 /particle/accuracy_4_of_4‚0.9655295610427856 ("Purity/1FRt2SRqqt0FBt/event_purity‚0 7 Purity/*FRt0SRqqt*FBt/FBt_purity‚0.9947766065597534 7 Purity/*FRt1SRqqt2FBt/FBt_purity‚0.9074073433876038 +jet/accuracy_2_of_3‚0.49928611516952515 9"Purity/*FRt*SRqqt2FBt/event_purity‚0.8554984331130981 9"Purity/1FRt*SRqqt2FBt/SRqqt_purity‚0.1111111119389534 :"Purity/1FRt1SRqqt*FBt/event_purity‚0.03299510478973389 8 Purity/2FRt2SRqqt*FBt/FRt_purity‚0.07629287242889404 =&Purity/*FRt*SRqqt1FBt/event_proportion‚0.0617094486951828 :"Purity/*FRt0SRqqt*FBt/event_purity‚0.20045611262321472 9"Purity/*FRt1SRqqt0FBt/SRqqt_purity‚0.6788672208786011 ("Purity/1FRt2SRqqt*FBt/event_purity‚0 =&Purity/2FRt0SRqqt*FBt/event_proportion‚0.0208565853536129 9"Purity/2FRt1SRqqt1FBt/SRqqt_purity‚0.7777777910232544 9"Purity/0FRt1SRqqt2FBt/SRqqt_purity‚0.1538461595773697 & Purity/2FRt0SRqqt1FBt/FBt_purity‚1 9"Purity/1FRt*SRqqt0FBt/SRqqt_purity‚0.7344927191734314 7 Purity/1FRt0SRqqt2FBt/FRt_purity‚0.8578799366950989 :"Purity/1FRt2SRqqt1FBt/SRqqt_purity‚0.32499998807907104 8 Purity/2FRt0SRqqt0FBt/FRt_purity‚0.13435089588165283 ("Purity/0FRt0SRqqt0FBt/event_purity‚1 'particle/f_score‚0.6459327936172485 @&Purity/*FRt2SRqqt1FBt/event_proportion‚0.0009998140158131719 8 Purity/*FRt2SRqqt1FBt/FRt_purity‚0.10000000149011612 7 Purity/0FRt*SRqqt2FBt/FBt_purity‚0.9804892539978027 7 Purity/1FRt*SRqqt*FBt/FBt_purity‚0.9968254566192627 7 Purity/1FRt0SRqqt2FBt/FBt_purity‚0.9970238208770752 8 Purity/2FRt2SRqqt0FBt/FRt_purity‚0.07629287242889404 ("Purity/0FRt1SRqqt2FBt/event_purity‚0 9"Purity/*FRt*SRqqt0FBt/event_purity‚0.2246999740600586 <&Purity/*FRt*SRqqt0FBt/event_proportion‚0.881045401096344 :"Purity/*FRt*SRqqt2FBt/SRqqt_purity‚0.12962964177131653 7 Purity/*FRt1SRqqt2FBt/FRt_purity‚0.8500000238418579 ("Purity/*FRt2SRqqt1FBt/event_purity‚0 9"Purity/1FRt*SRqqt2FBt/event_purity‚0.7141912579536438 @&Purity/2FRt0SRqqt1FBt/event_proportion‚0.0007672990905120969 =&Purity/0FRt*SRqqt*FBt/event_proportion‚0.2839471697807312 =&Purity/0FRt*SRqqt0FBt/event_proportion‚0.2322126179933548 @&Purity/1FRt1SRqqt2FBt/event_proportion‚0.0002325148816453293 8 Purity/2FRt0SRqqt*FBt/FRt_purity‚0.15624406933784485 7 Purity/*FRt2SRqqt*FBt/FBt_purity‚0.9814814329147339 7 Purity/0FRt0SRqqt*FBt/FBt_purity‚0.9934538006782532 9"Purity/0FRt2SRqqt0FBt/event_purity‚0.3490021228790283 6 Purity/1FRt*SRqqt1FBt/FRt_purity‚0.547100841999054 @&Purity/1FRt*SRqqt2FBt/event_proportion‚0.0027204242069274187 7 Purity/1FRt1SRqqt2FBt/FBt_purity‚0.8888888955116272 +jet/accuracy_3_of_4‚0.08986274152994156 9"Purity/0FRt*SRqqt2FBt/SRqqt_purity‚0.1538461595773697 9"Purity/0FRt*SRqqt0FBt/event_purity‚0.5959671139717102 & Purity/2FRt*SRqqt2FBt/FBt_purity‚1 jet/accuracy_2_of_5‚1 7 Purity/*FRt*SRqqt1FBt/FBt_purity‚0.9976805448532104 ,&Purity/*FRt2SRqqt2FBt/event_proportion‚0 8 Purity/*FRt*SRqqt0FBt/FRt_purity‚0.11705125123262405 ("Purity/*FRt1SRqqt2FBt/event_purity‚0 , Purity/0FRt2SRqqt*FBt/FBt_purity‚0.96875 9"Purity/2FRt*SRqqt*FBt/SRqqt_purity‚0.7455062866210938 9"Purity/*FRt*SRqqt1FBt/event_purity‚0.7244147062301636 :"Purity/*FRt1SRqqt2FBt/SRqqt_purity‚0.12962964177131653 8&Purity/*FRt2SRqqt0FBt/event_proportion‚ 0.05322265625 6validation_average_jet_accuracy‚0.3354771137237549 epoch‚14 )loss/val_total_loss‚9.421729369220609 6 Purity/*FRt0SRqqt1FBt/FRt_purity‚0.611768364906311 particle/accuracy_1_of_2‚1 ?&Purity/0FRt*SRqqt2FBt/event_proportion‚0.008068266324698925 ?&Purity/0FRt0SRqqt2FBt/event_proportion‚0.007626487873494625 @&Purity/1FRt0SRqqt2FBt/event_proportion‚0.0024879092816263437 9"Purity/1FRt1SRqqt1FBt/SRqqt_purity‚0.6772553324699402 7 Purity/1FRt1SRqqt2FBt/FRt_purity‚0.8888888955116272 =&Purity/2FRt*SRqqt0FBt/event_proportion‚0.0382486991584301 + Purity/2FRt*SRqqt2FBt/FRt_purity‚0.6875 8 Purity/*FRt*SRqqt*FBt/FRt_purity‚0.13206155598163605 9"Purity/0FRt*SRqqt2FBt/event_purity‚0.9129493236541748 9"Purity/*FRt*SRqqt0FBt/SRqqt_purity‚0.6716972589492798 7 Purity/0FRt0SRqqt2FBt/FBt_purity‚0.9849433898925781 >&Purity/0FRt2SRqqt0FBt/event_proportion‚0.02385602705180645 7 Purity/1FRt0SRqqt1FBt/FRt_purity‚0.6188405752182007 7 Purity/1FRt1SRqqt*FBt/FBt_purity‚0.9934239983558655 9"Purity/1FRt1SRqqt1FBt/event_purity‚0.2664455473423004 A&Purity/1FRt2SRqqt1FBt/event_proportion‚0.00027901786961592734 & Purity/2FRt*SRqqt*FBt/FBt_purity‚1 9"Purity/0FRt0SRqqt*FBt/event_purity‚0.9920435547828674 *jet/accuracy_2_of_4‚0.7122591137886047 8"Purity/*FRt*SRqqt1FBt/SRqqt_purity‚0.582953691482544 ?&Purity/0FRt1SRqqt1FBt/event_proportion‚0.011555989272892475 /particle/accuracy_3_of_4‚0.9965986609458923 ("Purity/2FRt*SRqqt2FBt/SRqqt_purity‚0 =&Purity/*FRt0SRqqt1FBt/event_proportion‚0.0418526791036129 :"Purity/*FRt1SRqqt0FBt/event_purity‚0.36378487944602966 =&Purity/0FRt1SRqqt*FBt/event_proportion‚0.2203543484210968 ,"Purity/2FRt*SRqqt2FBt/event_purity‚0.375 _runtime‚4167.517203545 8 Purity/*FRt1SRqqt1FBt/FRt_purity‚0.45688071846961975 9"Purity/0FRt0SRqqt2FBt/event_purity‚0.9698869585990906 :"Purity/1FRt*SRqqt0FBt/event_purity‚0.09697233885526657JÓ€²Ü"µ„| 7 Purity/0FRt*SRqqt*FBt/FBt_purity‚0.9921701550483704 7 Purity/1FRt*SRqqt2FBt/FBt_purity‚0.9894558191299438 7 Purity/1FRt0SRqqt*FBt/FBt_purity‚0.9988385438919067 8 Purity/1FRt1SRqqt1FBt/FRt_purity‚0.44262850284576416 ("Purity/1FRt2SRqqt1FBt/event_purity‚0 7 Purity/2FRt*SRqqt1FBt/FRt_purity‚0.5358333587646484 ;"Purity/2FRt0SRqqt0FBt/event_purity‚0.020245129242539406 8 Purity/2FRt1SRqqt*FBt/FRt_purity‚0.18487510085105896 7 Purity/*FRt0SRqqt2FBt/FRt_purity‚0.8468632698059082 =&Purity/0FRt*SRqqt1FBt/event_proportion‚0.0436662957072258 9"Purity/0FRt2SRqqt0FBt/SRqqt_purity‚0.5888653993606567 ,&Purity/2FRt2SRqqt1FBt/event_proportion‚0 9"Purity/0FRt*SRqqt0FBt/SRqqt_purity‚0.6178103685379028 =&Purity/0FRt0SRqqt*FBt/event_proportion‚0.0390159972012043 9"Purity/1FRt1SRqqt*FBt/SRqqt_purity‚0.7467018365859985 <&Purity/2FRt*SRqqt*FBt/event_proportion‚0.040062315762043 7 Purity/*FRt0SRqqt1FBt/FBt_purity‚0.9981421232223511 =&Purity/0FRt0SRqqt1FBt/event_proportion‚0.0313895083963871 9"Purity/0FRt1SRqqt0FBt/SRqqt_purity‚0.6245434284210205 @&Purity/0FRt1SRqqt2FBt/event_proportion‚0.0004417782765813172 >&Purity/0FRt2SRqqt*FBt/event_proportion‚0.02457682229578495 @&Purity/0FRt2SRqqt1FBt/event_proportion‚0.0007207961170934141 7 Purity/1FRt*SRqqt*FBt/FRt_purity‚0.1296166181564331 @&Purity/2FRt1SRqqt2FBt/event_proportion‚2.325148852833081e-05 8 Purity/*FRt0SRqqt0FBt/FRt_purity‚0.12736909091472626 =&Purity/0FRt1SRqqt0FBt/event_proportion‚0.2083565890789032 & Purity/2FRt1SRqqt2FBt/FBt_purity‚1 :"Purity/*FRt1SRqqt1FBt/event_purity‚0.40683332085609436 :"Purity/*FRt2SRqqt*FBt/event_purity‚0.15227782726287842 9"Purity/0FRt1SRqqt*FBt/SRqqt_purity‚0.6196860074996948 >&Purity/1FRt2SRqqt*FBt/event_proportion‚0.02322823740541935 & Purity/2FRt0SRqqt*FBt/FBt_purity‚1 @&Purity/2FRt*SRqqt1FBt/event_proportion‚0.0016043527284637094 @&Purity/2FRt1SRqqt1FBt/event_proportion‚0.0008370535797439516 =&Purity/*FRt*SRqqt*FBt/event_proportion‚0.9537528157234192 =&Purity/*FRt1SRqqt*FBt/event_proportion‚0.3866024911403656 =&Purity/1FRt*SRqqt0FBt/event_proportion‚0.6105840802192688 8 Purity/1FRt*SRqqt0FBt/FRt_purity‚0.11510719358921051 :"Purity/1FRt1SRqqt0FBt/event_purity‚0.02252895012497902 9"Purity/2FRt2SRqqt*FBt/SRqqt_purity‚0.7490408420562744 particle/accuracy_2_of_2‚1 (particle/accuracy‚0.8498379588127136 8 Purity/*FRt0SRqqt*FBt/FRt_purity‚0.14258141815662384 & Purity/1FRt*SRqqt1FBt/FBt_purity‚1 *jet/accuracy_1_of_4‚0.9491969347000122 9"Purity/2FRt*SRqqt1FBt/event_purity‚0.0950000062584877 9"Purity/2FRt1SRqqt*FBt/SRqqt_purity‚0.7434011697769165 ( Purity/2FRt1SRqqt2FBt/FRt_purity‚0.5 trainer/global_step‚5999 7 Purity/*FRt*SRqqt1FBt/FRt_purity‚0.5424790382385254 9"Purity/2FRt0SRqqt1FBt/event_purity‚0.2083333432674408 7 Purity/*FRt2SRqqt1FBt/FBt_purity‚0.9814814329147339 7 Purity/0FRt0SRqqt1FBt/FBt_purity‚0.9974045753479004 =&Purity/1FRt0SRqqt*FBt/event_proportion‚0.4530552327632904 particle/accuracy_2_of_4‚1 ("Purity/2FRt1SRqqt2FBt/event_purity‚0 9"Purity/*FRt1SRqqt*FBt/SRqqt_purity‚0.6741831302642822 8 Purity/*FRt2SRqqt0FBt/FRt_purity‚0.07300971448421478 ."Purity/0FRt2SRqqt1FBt/SRqqt_purity‚0.40625 9"Purity/1FRt*SRqqt*FBt/SRqqt_purity‚0.7306381464004517 ("Purity/1FRt1SRqqt2FBt/event_purity‚0 8"Purity/1FRt2SRqqt0FBt/SRqqt_purity‚0.681550920009613 6 Purity/*FRt*SRqqt*FBt/FBt_purity‚0.993619441986084 9"Purity/0FRt*SRqqt1FBt/SRqqt_purity‚0.5393243432044983 9"Purity/1FRt0SRqqt1FBt/event_purity‚0.5834558010101318 ;"Purity/2FRt*SRqqt0FBt/event_purity‚0.010464348830282688 <&Purity/*FRt2SRqqt*FBt/event_proportion‚0.054222472012043 =&Purity/0FRt0SRqqt0FBt/event_proportion‚0.0462472103536129 , Purity/0FRt2SRqqt1FBt/FBt_purity‚0.96875 9"Purity/*FRt0SRqqt1FBt/event_purity‚0.8855803608894348 9"Purity/0FRt*SRqqt*FBt/event_purity‚0.6443917155265808 ,&Purity/1FRt2SRqqt2FBt/event_proportion‚0 5 Purity/*FRt2SRqqt*FBt/FRt_purity‚0.07302226126194 9"Purity/0FRt2SRqqt*FBt/SRqqt_purity‚0.5835638046264648 ,&Purity/0FRt2SRqqt2FBt/event_proportion‚0 ("Purity/2FRt1SRqqt0FBt/event_purity‚0 ("Purity/2FRt2SRqqt0FBt/event_purity‚0 particle/accuracy_1_of_3‚1 :"Purity/*FRt*SRqqt*FBt/event_purity‚0.26439911127090454 9"Purity/*FRt0SRqqt2FBt/event_purity‚0.9150582551956177 =&Purity/*FRt1SRqqt1FBt/event_proportion‚0.0188569575548172 9"Purity/0FRt0SRqqt1FBt/event_purity‚0.9974045753479004 9"Purity/0FRt1SRqqt1FBt/event_purity‚0.5165648460388184 :"Purity/1FRt*SRqqt1FBt/event_purity‚0.45168453454971313 9"Purity/1FRt1SRqqt2FBt/SRqqt_purity‚0.1111111119389534 7 Purity/0FRt1SRqqt2FBt/FBt_purity‚0.9230769276618958 9"Purity/*FRt2SRqqt0FBt/SRqqt_purity‚0.6471351981163025 7 Purity/2FRt0SRqqt1FBt/FRt_purity‚0.5885416269302368 ("Purity/2FRt1SRqqt2FBt/SRqqt_purity‚0 particle/accuracy_3_of_5‚1 9"Purity/*FRt*SRqqt*FBt/SRqqt_purity‚0.6671638488769531 :"Purity/*FRt0SRqqt0FBt/event_purity‚0.12221173942089081 9"Purity/0FRt1SRqqt1FBt/SRqqt_purity‚0.5571209192276001 9"Purity/1FRt*SRqqt1FBt/SRqqt_purity‚0.6540589928627014 =&Purity/1FRt1SRqqt*FBt/event_proportion‚0.1534598171710968 & Purity/1FRt1SRqqt1FBt/FBt_purity‚1 8&Purity/1FRt2SRqqt0FBt/event_proportion‚ 0.02294921875 & Purity/2FRt*SRqqt1FBt/FBt_purity‚1 >&Purity/1FRt*SRqqt1FBt/event_proportion‚0.01643880270421505 8 Purity/1FRt1SRqqt0FBt/FRt_purity‚0.08636563271284103 8 Purity/2FRt*SRqqt0FBt/FRt_purity‚0.13310836255550385 @&Purity/2FRt*SRqqt2FBt/event_proportion‚0.0002092633949359879 ?&Purity/2FRt2SRqqt0FBt/event_proportion‚0.006417410913854837 jet/accuracy_3_of_5‚1 particle/accuracy_1_of_5‚1 particle/accuracy_2_of_5‚1 7 Purity/*FRt*SRqqt2FBt/FBt_purity‚0.9822051525115967 =&Purity/*FRt0SRqqt2FBt/event_proportion‚0.0103004090487957 8"Purity/0FRt*SRqqt*FBt/SRqqt_purity‚0.612989068031311 9"Purity/1FRt1SRqqt0FBt/SRqqt_purity‚0.7509251832962036 A&Purity/2FRt0SRqqt2FBt/event_proportion‚0.00018601190822664648 particle/accuracy_1_of_1‚1 *particle/specificity‚0.907795786857605 _step‚494 :"Purity/0FRt2SRqqt*FBt/event_purity‚0.33906635642051697 8 Purity/1FRt2SRqqt*FBt/FRt_purity‚0.07167483866214752 +jet/accuracy_1_of_1‚0.28660327196121216 6 Purity/*FRt*SRqqt2FBt/FRt_purity‚0.843461811542511 9"Purity/0FRt*SRqqt1FBt/event_purity‚0.8508739471435547 =&Purity/*FRt1SRqqt0FBt/event_proportion‚0.3670479953289032 8 Purity/1FRt1SRqqt*FBt/FRt_purity‚0.10316571593284607 7 Purity/2FRt*SRqqt*FBt/FRt_purity‚0.1519196480512619 8 Purity/2FRt1SRqqt1FBt/FRt_purity‚0.48842594027519226 particle/accuracy_2_of_3‚1 7 Purity/0FRt1SRqqt*FBt/FBt_purity‚0.9894083142280579 9"Purity/0FRt1SRqqt0FBt/event_purity‚0.6245434284210205 9"Purity/1FRt0SRqqt2FBt/event_purity‚0.7826153039932251 & Purity/1FRt2SRqqt1FBt/FBt_purity‚1 =&Purity/*FRt0SRqqt0FBt/event_proportion‚0.4607747495174408 :"Purity/*FRt1SRqqt*FBt/event_purity‚0.36516764760017395 ("Purity/0FRt2SRqqt1FBt/event_purity‚0 :"Purity/1FRt*SRqqt*FBt/event_purity‚0.10889540612697601 7 Purity/1FRt2SRqqt0FBt/FRt_purity‚0.0717899352312088 +jet/accuracy_3_of_3‚0.06693118810653687 jet/accuracy_4_of_5‚0 7 Purity/*FRt1SRqqt1FBt/FBt_purity‚0.9974919557571411 =&Purity/1FRt*SRqqt*FBt/event_proportion‚0.6297432780265808 8 Purity/1FRt0SRqqt*FBt/FRt_purity‚0.14144790172576904 @&Purity/1FRt1SRqqt1FBt/event_proportion‚0.0064639137126505375 9"Purity/2FRt*SRqqt1FBt/SRqqt_purity‚0.7777777910232544 ("Purity/2FRt1SRqqt*FBt/event_purity‚0 & Purity/2FRt1SRqqt*FBt/FBt_purity‚1 7 Purity/*FRt1SRqqt0FBt/FRt_purity‚0.0972537100315094 :"Purity/*FRt2SRqqt0FBt/event_purity‚0.15519270300865173 =&Purity/2FRt1SRqqt*FBt/event_proportion‚0.0127883180975914 @&Purity/*FRt1SRqqt2FBt/event_proportion‚0.0006975446594879031 ("Purity/2FRt2SRqqt*FBt/event_purity‚0 ?&Purity/2FRt2SRqqt*FBt/event_proportion‚0.006417410913854837 *jet/accuracy_1_of_3‚0.9059290885925293 ,jet/accuracy_4_of_4‚0.013899499550461769 & Purity/2FRt0SRqqt2FBt/FBt_purity‚1 particle/accuracy_4_of_5‚1 9"Purity/2FRt1SRqqt0FBt/SRqqt_purity‚0.7423023581504822 9"Purity/2FRt2SRqqt0FBt/SRqqt_purity‚0.7490408420562744 9"Purity/2FRt0SRqqt2FBt/event_purity‚0.4285714328289032 ?&Purity/*FRt*SRqqt2FBt/event_proportion‚0.010997953824698925 9"Purity/*FRt1SRqqt1FBt/SRqqt_purity‚0.6044234037399292 9"Purity/*FRt2SRqqt*FBt/SRqqt_purity‚0.6422409415245056 9"Purity/0FRt1SRqqt*FBt/event_purity‚0.6172688603401184 :"Purity/1FRt0SRqqt0FBt/event_purity‚0.12686528265476227 8 Purity/1FRt0SRqqt0FBt/FRt_purity‚0.12686528265476227 9"Purity/2FRt*SRqqt0FBt/SRqqt_purity‚0.7450370788574219 7 Purity/2FRt0SRqqt2FBt/FRt_purity‚0.7142857313156128 8 Purity/*FRt1SRqqt*FBt/FRt_purity‚0.1149044260